What is wrong while providing arguments for sws_scale? - ffmpeg

In the following code, I can't figure out what's wrong:
uint8_t *dstData[4];
int dstLinesize[4];
AVPixelFormat convertToPixFmt = AV_PIX_FMT_RGBA;
int ret;
// ...
printf("tmp_frame format: %d (%s) %dx%d\n", tmp_frame->format, av_get_pix_fmt_name((AVPixelFormat)tmp_frame->format), tmp_frame->width, tmp_frame->height);
// The above line prints: tmp_frame format: 23 (nv12) 480x480
int size = av_image_get_buffer_size(convertToPixFmt, tmp_frame->width, tmp_frame->height, 1);
uint8_t *buffer = (uint8_t *) av_malloc(size);
ret = av_image_copy_to_buffer(buffer, size,
(const uint8_t * const *)&tmp_frame->data[i],
(const int *)&tmp_frame->linesize[i], (AVPixelFormat)tmp_frame->format,
tmp_frame->width, tmp_frame->height, 1);
ASSERT(ret >= 0);
ret = av_image_fill_arrays(dstData, dstLinesize, buffer, convertToPixFmt, dest_width, dest_height, 1);
ASSERT(ret >= 0);
ret = sws_scale(
convertContext,
dstData,
dstLinesize,
0,
dest_width,
convertedFrame->data,
convertedFrame->linesize);
printf("sws_scale returns %d\n", ret); // prints: sws_scale returns 0
ASSERT(ret == tmp_frame->height);
// ...
It's part of a code which uses dxva2 to obtain tmp_frame. I inspired the code from hw_decode.c and am sure that there's no mistake in the code. The tmp_frame is properly made in NV12 format. The error occurs just when I call sws_scale and it's:
bad src image pointers
So I don't know how to provide pointers not to get this error and sws_scale may work properly.
Any idea?
I update the question to include my whole code:
static AVBufferRef *hw_device_ctx = NULL;
static enum AVPixelFormat hw_pix_fmt;
static FILE *output_file = NULL;
int main(int argc, char *argv[])
{
AVFormatContext *input_ctx = NULL;
int video_stream, ret;
AVStream *video = NULL;
AVCodecContext *decoder_ctx = NULL;
AVCodec *decoder = NULL;
AVPacket packet;
enum AVHWDeviceType type;
int i;
if (argc < 2)
{
fprintf(stderr, "Usage: %s <input file>\n", argv[0]);
return -1;
}
type = av_hwdevice_find_type_by_name("dxva2");
ASSERT(type != AV_HWDEVICE_TYPE_NONE);
ASSERT(avformat_open_input(&input_ctx, argv[1], NULL, NULL) == 0);
ASSERT(avformat_find_stream_info(input_ctx, NULL) >= 0);
video_stream = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &decoder, 0);
ASSERT(video_stream >= 0);
decoder_ctx = avcodec_alloc_context3(decoder);
ASSERT(decoder_ctx);
video = input_ctx->streams[video_stream];
ASSERT(avcodec_parameters_to_context(decoder_ctx, video->codecpar) >= 0);
ASSERT(av_hwdevice_ctx_create(&hw_device_ctx, type, NULL, NULL, 0) >= 0);
decoder_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
ASSERT(avcodec_open2(decoder_ctx, decoder, NULL) >= 0);
printf("video info: %dx%d\n", decoder_ctx->width, decoder_ctx->height);
AVFrame *frame = av_frame_alloc();
ASSERT(frame);
AVFrame *sw_frame = av_frame_alloc();
ASSERT(sw_frame);
AVFrame* convertedFrame = av_frame_alloc();
ASSERT(convertedFrame);
AVPixelFormat convertToPixFmt = AV_PIX_FMT_RGBA;
//int dest_width = 320, dest_height = 200;
int dest_width = decoder_ctx->width, dest_height = decoder_ctx->height;
SwsContext* convertContext = sws_getContext(decoder_ctx->width, decoder_ctx->height, AV_PIX_FMT_YUV420P,
dest_width, dest_height, convertToPixFmt,
SWS_FAST_BILINEAR, NULL, NULL, NULL);
ASSERT(convertContext);
int convertedFrameAspectBufferSize = avpicture_get_size(convertToPixFmt, dest_width, dest_height);
void *convertedFrameBuffer = av_malloc(convertedFrameAspectBufferSize);
avpicture_fill((AVPicture*)convertedFrame, (uint8_t *)convertedFrameBuffer, convertToPixFmt, dest_width, dest_height);
output_file = fopen("1.out", "w+");
for (int i = 0; /*i < 20*/; i++)
{
ret = av_read_frame(input_ctx, &packet);
if (ret == AVERROR_EOF)
break;
ASSERT(ret >= 0);
if (video_stream != packet.stream_index)
continue;
int ret = avcodec_send_packet(decoder_ctx, &packet);
ASSERT(ret >= 0);
//printf("%p", decoder->hw_configs->hwaccel);
ret = avcodec_receive_frame(decoder_ctx, frame);
if (ret < 0)
printf("%d\t%d\n", i, ret);
AVFrame *tmp_frame;
if (frame->format > 0) // hw enabled
{
ASSERT(av_hwframe_transfer_data(sw_frame, frame, 0) >= 0);
tmp_frame = sw_frame;
}
else
{
tmp_frame = frame;
}
printf("frame format: %d (%s) %dx%d\n", frame->format, av_get_pix_fmt_name((AVPixelFormat)frame->format), frame->width, frame->height);
printf("sw_frame format: %d (%s) %dx%d\n", sw_frame->format, av_get_pix_fmt_name((AVPixelFormat)sw_frame->format), sw_frame->width, sw_frame->height);
printf("tmp_frame format: %d (%s) %dx%d\n", tmp_frame->format, av_get_pix_fmt_name((AVPixelFormat)tmp_frame->format), tmp_frame->width, tmp_frame->height);
/*
video info: 480x480
frame format: 53 (dxva2_vld) 480x480
sw_frame format: 23 (nv12) 480x480
[swscaler # 004cb2c0] bad src image pointers
*/
int size = av_image_get_buffer_size(convertToPixFmt, tmp_frame->width, tmp_frame->height, 1);
uint8_t *buffer = (uint8_t *) av_malloc(size);
ret = av_image_copy_to_buffer(buffer, size,
(const uint8_t * const *)&tmp_frame->data[i],
(const int *)&tmp_frame->linesize[i], (AVPixelFormat)tmp_frame->format,
tmp_frame->width, tmp_frame->height, 1);
ASSERT(ret > 0);
ret = av_image_fill_arrays(dstData, dstLinesize, buffer, convertToPixFmt, dest_width, dest_height, 1);
ASSERT(ret > 0);
ret = sws_scale(
convertContext,
tmp_frame->data,
tmp_frame->linesize,
0,
dest_width,
convertedFrame->data,
convertedFrame->linesize);
printf("sws_scale returns %d\n", ret);
ASSERT(ret == tmp_frame->height);
ret = fwrite(convertedFrame->data, tmp_frame->height * tmp_frame->width, 1, output_file);
ASSERT(ret == 1);
break;
}
av_frame_free(&frame);
av_packet_unref(&packet);
avcodec_free_context(&decoder_ctx);
avformat_close_input(&input_ctx);
av_buffer_unref(&hw_device_ctx);
return 0;
}

Related

Saving frames as JPG with FFMPEG (Visual Studio / C++)

I am trying to save all frames from a mp4 video in separate JPG files, I have a code that runs and actually saves something to JPG files but files are not recognized as images and nothing is showing.
Below my full code, I am using Visual Studio 2022 in Windows 11 and FFMPEG 5.1. The function that saves the images is save_frame_as_jpeg which is actually an adaption from the code provided here but changing the use of avcodec_encode_video2 for avcodec_send_frame/avcodec_receive_packet as indicated in the documentation.
I am obiously doing something wrong but cannot quite find it, BTW, I know that a simple command (ffmpeg -i input.mp4 -vf fps=1 vid_%d.png) will do this but I am requiring to do it by code.
Any help is appreciated, thanks in advance!
// FfmpegTests.cpp : This file contains the 'main' function. Program execution begins and ends there.
//
#pragma warning(disable : 4996)
extern "C"
{
#include "libavformat/avformat.h"
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavutil/opt.h"
#include "libavutil/avutil.h"
#include "libavutil/error.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/buffersink.h"
#include "libswscale/swscale.h"
}
#pragma comment(lib, "avcodec.lib")
#pragma comment(lib, "avformat.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "avutil.lib")
#pragma comment(lib, "swscale.lib")
#include <cstdio>
#include <iostream>
#include <chrono>
#include <thread>
static AVFormatContext* fmt_ctx;
static AVCodecContext* dec_ctx;
AVFilterGraph* filter_graph;
AVFilterContext* buffersrc_ctx;
AVFilterContext* buffersink_ctx;
static int video_stream_index = -1;
const char* filter_descr = "scale=78:24,transpose=cclock";
static int64_t last_pts = AV_NOPTS_VALUE;
static int open_input_file(const char* filename)
{
const AVCodec* dec;
int ret;
if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
/* select the video stream */
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
return ret;
}
video_stream_index = ret;
/* create decoding context */
dec_ctx = avcodec_alloc_context3(dec);
if (!dec_ctx)
return AVERROR(ENOMEM);
avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);
/* init the video decoder */
if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
return ret;
}
return 0;
}
static int init_filters(const char* filters_descr)
{
char args[512];
int ret = 0;
const AVFilter* buffersrc = avfilter_get_by_name("buffer");
const AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs = avfilter_inout_alloc();
AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static void display_frame(const AVFrame* frame, AVRational time_base)
{
int x, y;
uint8_t* p0, * p;
int64_t delay;
if (frame->pts != AV_NOPTS_VALUE) {
if (last_pts != AV_NOPTS_VALUE) {
/* sleep roughly the right amount of time;
* usleep is in microseconds, just like AV_TIME_BASE. */
AVRational timeBaseQ;
timeBaseQ.num = 1;
timeBaseQ.den = AV_TIME_BASE;
delay = av_rescale_q(frame->pts - last_pts, time_base, timeBaseQ);
if (delay > 0 && delay < 1000000)
std::this_thread::sleep_for(std::chrono::microseconds(delay));
}
last_pts = frame->pts;
}
/* Trivial ASCII grayscale display. */
p0 = frame->data[0];
puts("\033c");
for (y = 0; y < frame->height; y++) {
p = p0;
for (x = 0; x < frame->width; x++)
putchar(" .-+#"[*(p++) / 52]);
putchar('\n');
p0 += frame->linesize[0];
}
fflush(stdout);
}
int save_frame_as_jpeg(AVCodecContext* pCodecCtx, AVFrame* pFrame, int FrameNo) {
int ret = 0;
const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_JPEG2000);
if (!jpegCodec) {
return -1;
}
AVCodecContext* jpegContext = avcodec_alloc_context3(jpegCodec);
if (!jpegContext) {
return -1;
}
jpegContext->pix_fmt = pCodecCtx->pix_fmt;
jpegContext->height = pFrame->height;
jpegContext->width = pFrame->width;
jpegContext->time_base = AVRational{ 1,10 };
ret = avcodec_open2(jpegContext, jpegCodec, NULL);
if (ret < 0) {
return ret;
}
FILE* JPEGFile;
char JPEGFName[256];
AVPacket packet;
packet.data = NULL;
packet.size = 0;
av_init_packet(&packet);
int gotFrame;
ret = avcodec_send_frame(jpegContext, pFrame);
if (ret < 0) {
return ret;
}
ret = avcodec_receive_packet(jpegContext, &packet);
if (ret < 0) {
return ret;
}
sprintf(JPEGFName, "c:\\folder\\dvr-%06d.jpg", FrameNo);
JPEGFile = fopen(JPEGFName, "wb");
fwrite(packet.data, 1, packet.size, JPEGFile);
fclose(JPEGFile);
av_packet_unref(&packet);
avcodec_close(jpegContext);
return 0;
}
int main(int argc, char** argv)
{
AVFrame* frame;
AVFrame* filt_frame;
AVPacket* packet;
int ret;
if (argc != 2) {
fprintf(stderr, "Usage: %s file\n", argv[0]);
exit(1);
}
frame = av_frame_alloc();
filt_frame = av_frame_alloc();
packet = av_packet_alloc();
if (!frame || !filt_frame || !packet) {
fprintf(stderr, "Could not allocate frame or packet\n");
exit(1);
}
if ((ret = open_input_file(argv[1])) < 0)
goto end;
if ((ret = init_filters(filter_descr)) < 0)
goto end;
while (true)
{
if ((ret = av_read_frame(fmt_ctx, packet)) < 0)
break;
if (packet->stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0)
{
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
frame->pts = frame->best_effort_timestamp;
/* push the decoded frame into the filtergraph */
if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
break;
}
/* pull filtered frames from the filtergraph */
while (1) {
ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0)
goto end;
display_frame(filt_frame, buffersink_ctx->inputs[0]->time_base);
av_frame_unref(filt_frame);
ret = save_frame_as_jpeg(dec_ctx, frame, dec_ctx->frame_number);
if (ret < 0)
goto end;
}
av_frame_unref(frame);
}
}
av_packet_unref(packet);
}
end:
avfilter_graph_free(&filter_graph);
avcodec_free_context(&dec_ctx);
avformat_close_input(&fmt_ctx);
av_frame_free(&frame);
av_frame_free(&filt_frame);
av_packet_free(&packet);
if (ret < 0 && ret != AVERROR_EOF) {
char errBuf[AV_ERROR_MAX_STRING_SIZE]{0};
int res = av_strerror(ret, errBuf, AV_ERROR_MAX_STRING_SIZE);
fprintf(stderr, "Error: %s\n", errBuf);
exit(1);
}
exit(0);
}
Well nevermind, I just realized I had an error with the specified codec for JPEG decoding, if somenone's facing this issue you have to use:
*const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);*
instead of:
*const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_JPEG2000);*
and also add this line:
*jpegContext->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;*
before the call to avcodec_open2

FFMPEG transcoder producing broken video

I have the following code to transcode video which is closely related to the FFMPEG transcoding example.
However it produced broken video as shown:
It seems like the i-frames are decoded correctly, but the p and b frames are out of order? I thought av_interleaved_write_frame() would rectify that for me. It also seems like the libx264 is not creating any extradata which I thought it would.
Could someone please help me work out why?
Many thanks
#include <stdbool.h>
#include <stdio.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/mathematics.h>
#include <libavutil/opt.h>
#define DEBUG(level, format, ...) fprintf(stderr, format "\n", ##__VA_ARGS__)
#define stringify2(var) #var
#define stringify(var) stringify2(var)
#define RASSERT(cond, ...) do { \
if (!(cond)) { \
DEBUG(LOG_FATAL, __FILE__ ":" stringify(__LINE__) " " "Assertion failed! " #cond ". " __VA_ARGS__); \
abort(); \
} \
} while (0)
#define FFCHECK(ret, func) RASSERT(ret == 0, #func " failed: %s (%d)", av_err2str(ret), ret)
typedef struct decode_context {
AVFormatContext *format;
AVCodecContext *videoCodec;
AVCodecContext *audioCodec;
AVStream *videoStream;
AVStream *audioStream;
} decode_context_t;
typedef struct encode_context {
AVFormatContext *format;
AVCodecContext *videoCodec;
AVCodecContext *audioCodec;
AVStream *videoStream;
AVStream *audioStream;
} encode_context_t;
void open_input(decode_context_t *dec, const char *file) {
int ret;
ret = avformat_open_input(&dec->format, file, NULL, NULL);
FFCHECK(ret, "avformat_open_input()");
ret = avformat_find_stream_info(dec->format, NULL);
FFCHECK(ret, "avformat_find_stream_info()");
for (unsigned int i = 0; i < dec->format->nb_streams; ++i) {
AVStream * stream = dec->format->streams[i];
enum AVMediaType type = stream->codecpar->codec_type;
switch (type) {
case AVMEDIA_TYPE_VIDEO:
if (dec->videoStream)
break;
dec->videoStream = stream;
break;
case AVMEDIA_TYPE_AUDIO:
dec->audioStream = stream;
if (dec->audioStream)
break;
break;
default:
break;
}
}
RASSERT(dec->videoStream != NULL, "Didn't find video stream");
const AVCodec * codec = avcodec_find_decoder(dec->videoStream->codecpar->codec_id);
RASSERT(codec, "Failed to find decoder");
dec->videoCodec = avcodec_alloc_context3(codec);
RASSERT(dec->videoCodec, "avcodec_alloc_context3() failed");
ret = avcodec_parameters_to_context(dec->videoCodec, dec->videoStream->codecpar);
FFCHECK(ret, "avcodec_parameters_to_context()");
dec->videoCodec->framerate = av_guess_frame_rate(dec->format, dec->videoStream, NULL);
ret = avcodec_open2(dec->videoCodec, codec, NULL);
FFCHECK(ret, "avcodec_open2()");
}
void open_output(encode_context_t *enc, const char *file, decode_context_t *dec) {
int ret;
ret = avformat_alloc_output_context2(&enc->format, NULL, NULL, file);
FFCHECK(ret, "avformat_alloc_output_context2()");
enc->videoStream = avformat_new_stream(enc->format, NULL);
RASSERT(enc->videoStream, "avformat_new_stream() failed");
enc->videoStream->id = enc->format->nb_streams - 1;
const AVCodec *codec = avcodec_find_encoder_by_name("libx264");
RASSERT(codec, "Failed to find encoder");
enc->videoCodec = avcodec_alloc_context3(codec);
RASSERT(enc->videoCodec, "avcodec_alloc_context3() failed");
enc->videoCodec->bit_rate = 400000;
enc->videoCodec->width = dec->videoCodec->width;
enc->videoCodec->height = dec->videoCodec->height;
enc->videoCodec->sample_aspect_ratio = dec->videoCodec->sample_aspect_ratio;
enc->videoCodec->time_base = av_inv_q(dec->videoCodec->framerate);
//enc->videoCodec->gop_size = 12;
//enc->videoCodec->max_b_frames = 2;
enc->videoCodec->pix_fmt = dec->videoCodec->pix_fmt;
enc->videoCodec->framerate = dec->videoCodec->framerate;
if (codec->id == AV_CODEC_ID_H264) {
av_opt_set(enc->videoCodec->priv_data, "preset", "slow", 0);
av_opt_set(enc->videoCodec->priv_data, "profile", "high", 0);
av_opt_set(enc->videoCodec->priv_data, "level", "4.1", 0);
}
if (enc->format->flags & AVFMT_GLOBALHEADER)
enc->videoCodec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
ret = avcodec_open2(enc->videoCodec, codec, NULL);
FFCHECK(ret, "avcodec_open2()");
ret = avcodec_parameters_from_context(enc->videoStream->codecpar, enc->videoCodec);
FFCHECK(ret, "avcodec_parameters_from_context()");
//enc->videoStream->time_base = enc->videoCodec->time_base;
//enc->videoStream->codecpar->extradata = enc->videoCodec->extradata;
//enc->videoStream->codecpar->extradata_size = enc->videoCodec->extradata_size;
av_dump_format(enc->format, 0, file, 1);
ret = avio_open(&enc->format->pb, file, AVIO_FLAG_WRITE);
FFCHECK(ret, "avio_open()");
ret = avformat_write_header(enc->format, NULL);
FFCHECK(ret, "avformat_write_header()");
}
int main(int argc, const char * argv[]) {
int ret;
if (argc < 3) {
fprintf(stderr, "%s input output\n", argv[0]);
return 1;
}
decode_context_t dec_ctx = {};
decode_context_t *dec = &dec_ctx;
encode_context_t enc_ctx = {};
encode_context_t *enc = &enc_ctx;
open_input(dec, argv[1]);
open_output(enc, argv[2], dec);
while (true) {
AVPacket *packet = av_packet_alloc();
ret = av_read_frame(dec->format, packet);
if (ret < 0)
break;
if (packet->stream_index == dec->videoStream->index) {
ret = avcodec_send_packet(dec->videoCodec, packet);
if (ret == AVERROR(EAGAIN)) {
AVFrame * frame = av_frame_alloc();
while (true) {
ret = avcodec_receive_frame(dec->videoCodec, frame);
if (ret == AVERROR(EAGAIN))
break;
FFCHECK(ret, "avcodec_receive_frame()");
ret = avcodec_send_frame(enc->videoCodec, frame);
if (ret == AVERROR(EAGAIN)) {
AVPacket *pkt = av_packet_alloc();
while (true) {
ret = avcodec_receive_packet(enc->videoCodec, pkt);
if (ret == AVERROR(EAGAIN))
break;
FFCHECK(ret, "avcodec_receive_packet()");
pkt->stream_index = enc->videoStream->id;
av_packet_rescale_ts(pkt, dec->videoStream->time_base, enc->videoStream->time_base);
ret = av_interleaved_write_frame(enc->format, pkt);
FFCHECK(ret, "av_interleaved_write_frame()");
}
av_packet_free(&pkt);
}
}
av_frame_free(&frame);
} else {
FFCHECK(ret, "avcodec_send_packet()");
}
} else if (packet->stream_index == dec->audioStream->index) {
// Deal with audio
}
av_packet_free(&packet);
}
ret = av_write_trailer(enc->format);
FFCHECK(ret, "av_write_trailer()");
ret = avio_close(enc->format->pb);
FFCHECK(ret, "avio_close()");
// Close some more stuff
return 0;
}
After thinking about this some more, I realised that I wasn't even sure if my decoder was correct. And another thorough inspection of the transcoding example revealed a small difference in behaviour.
Effectively, I was doing this:
while (true) {
ret = avcodec_send_packet(dec->videoCodec, packet);
if (ret != AVERROR(EAGAIN)) {
continue;
}
ret = avcodec_receive_frame(dec->videoCodec, frame);
}
Removing the continue and immediately trying to receive frames from the decoder works much better, and fixed my issue.
while (true) {
ret = avcodec_send_packet(dec->videoCodec, packet);
if (ret != 0)
abort();
ret = avcodec_receive_frame(dec->videoCodec, frame);
}
I also had the same error in the encoding side so I've fixed it there too.

SIGABRT: abort attempting encoding PCM to AAC

I am trying to encoding incoming raw PCM audio data into an AAC encoded audio file. The following crashes with SIGABRT when it hits the avcodec_encode_audio2 call:
aac_encoding.c
#include <stdio.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswresample/swresample.h>
typedef struct AACEncoder {
AVFormatContext* pFormatCtx;
AVStream* audio_st;
AVCodecContext* pCodecCtx;
AVFrame* pFrame;
AVPacket* pkt;
uint8_t* frame_buf;
} AACEncoder;
AACEncoder *openEncoder(char* out_file) {
AACEncoder* encoder = (AACEncoder*)malloc(sizeof(AACEncoder*));
av_register_all();
AVFormatContext* pFormatCtx = avformat_alloc_context();
encoder->pFormatCtx = pFormatCtx;
AVOutputFormat* outFormat = av_guess_format(NULL, out_file, NULL);
pFormatCtx->oformat = outFormat;
if (avio_open(&pFormatCtx->pb, out_file, AVIO_FLAG_READ_WRITE) < 0) {
printf("Failed to open output file!\n");
return NULL;
}
AVStream* audio_st = avformat_new_stream(pFormatCtx, 0);
if (audio_st==NULL){
return NULL;
}
encoder->audio_st;
AVCodecContext* pCodecCtx = audio_st->codec;
encoder->pCodecCtx = pCodecCtx;
pCodecCtx->codec_id = outFormat->audio_codec;
pCodecCtx->codec_type = AVMEDIA_TYPE_AUDIO;
pCodecCtx->sample_fmt = AV_SAMPLE_FMT_FLTP;
pCodecCtx->sample_rate= 48000;
pCodecCtx->channel_layout = AV_CH_LAYOUT_MONO;
pCodecCtx->channels = av_get_channel_layout_nb_channels(pCodecCtx->channel_layout);
pCodecCtx->bit_rate = 64000;
av_dump_format(pFormatCtx, 0, out_file, 1);
AVCodec* pCodec = avcodec_find_encoder(pCodecCtx->codec_id);
if (!pCodec){
printf("Can not find encoder!\n");
return NULL;
}
if (avcodec_open2(pCodecCtx, pCodec,NULL) < 0){
printf("Failed to open encoder!\n");
return NULL;
}
AVFrame* pFrame = av_frame_alloc();
encoder->pFrame = pFrame;
pFrame->nb_samples= pCodecCtx->frame_size;
pFrame->format= pCodecCtx->sample_fmt;
int size = av_samples_get_buffer_size(NULL, pCodecCtx->channels,pCodecCtx->frame_size,pCodecCtx->sample_fmt, 1);
uint8_t* frame_buf = (uint8_t *)av_malloc(size);
encoder->frame_buf = frame_buf;
avcodec_fill_audio_frame(pFrame, pCodecCtx->channels, pCodecCtx->sample_fmt,(const uint8_t*)frame_buf, size, 1);
//Write Header
avformat_write_header(pFormatCtx,NULL);
AVPacket pkt;
encoder->pkt = &pkt;
av_new_packet(&pkt,size);
return encoder;
}
int writePCM(AACEncoder* encoder, int16_t* pcmData, size_t pcmSize) {
SwrContext* swr = swr_alloc();
av_opt_set_int(swr, "in_channel_layout", encoder->pCodecCtx->channel_layout, 0);
av_opt_set_int(swr, "out_channel_layout", encoder->pCodecCtx->channel_layout, 0);
av_opt_set_int(swr, "in_sample_rate", encoder->pCodecCtx->sample_rate, 0);
av_opt_set_int(swr, "out_sample_rate", encoder->pCodecCtx->sample_rate, 0);
av_opt_set_sample_fmt(swr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_FLT, 0);
swr_init(swr);
printf("Initialized SwrContext\n");
fflush(stdout);
int ret = swr_convert(swr, encoder->pFrame->extended_data, pcmSize, &pcmData, pcmSize);
int got_frame=0;
if(ret < 0){
printf("Failed to resample!\n");
return -1;
}
//Encode
ret = avcodec_encode_audio2(encoder->pCodecCtx, encoder->pkt, encoder->pFrame, &got_frame);
printf("Encoded audio using AAC\n");
fflush(stdout);
swr_free(&swr);
if(ret < 0){
printf("Failed to encode!\n");
return -1;
}
if (got_frame==1){
printf("Succeed to encode 1 frame! \tsize:%5d\n", encoder->pkt->size);
encoder->pkt->stream_index = encoder->audio_st->index;
ret = av_write_frame(encoder->pFormatCtx, encoder->pkt);
av_free_packet(encoder->pkt);
}
}

How to get picture buffer data in ffmpeg?

I'm trying to pass bitmap from ffmpeg to android.
It already works but it's displaying picture right on surface passed from java to native code.
How can i get frame buffer bitmap data to pass it to java?
I've tried to save out_frame buffer data:
unsigned char bmpFileHeader[14] = {'B', 'M', 0,0,0,0, 0,0, 0,0, 54, 0,0,0};
unsigned char bmpInfoHeader[40] = {40,0,0,0, 0,0,0,0, 0,0,0,0, 1,0, 24,0};
unsigned char bmpPad[3] = {0, 0, 0};
void saveBuffer(int fileIndex, int width, int height, unsigned char *buffer, int buffer_size) {
unsigned char filename[1024];
sprintf(filename, "/storage/sdcard0/3d_player_%d.bmp", fileIndex);
LOGI(10, "saving ffmpeg bitmap file: %d to %s", fileIndex, filename);
FILE *bitmapFile = fopen(filename, "wb");
if (!bitmapFile) {
LOGE(10, "failed to create ffmpeg bitmap file");
return;
}
unsigned char filesize = 54 + 3 * width * height; // 3 = (r,g,b)
bmpFileHeader[2] = (unsigned char)(filesize);
bmpFileHeader[3] = (unsigned char)(filesize >> 8);
bmpFileHeader[4] = (unsigned char)(filesize >> 16);
bmpFileHeader[5] = (unsigned char)(filesize >> 24);
bmpInfoHeader[4] = (unsigned char)(width);
bmpInfoHeader[5] = (unsigned char)(width >> 8);
bmpInfoHeader[6] = (unsigned char)(width >> 16);
bmpInfoHeader[7] = (unsigned char)(width >> 24);
bmpInfoHeader[8] = (unsigned char)(height);
bmpInfoHeader[9] = (unsigned char)(height >> 8);
bmpInfoHeader[10] = (unsigned char)(height >> 16);
bmpInfoHeader[11] = (unsigned char)(height >> 24);
fwrite(bmpFileHeader, 1, 14, bitmapFile);
fwrite(bmpInfoHeader, 1, 40, bitmapFile);
int i;
for (i=0; i<height; i++) {
fwrite(buffer + width * (height - 1) * 3, 3, width, bitmapFile);
fwrite(bmpPad, 1, (4-(width * 3) % 4) % 4, bitmapFile);
}
fflush(bitmapFile);
fclose(bitmapFile);
}
int player_decode_video(struct DecoderData * decoder_data, JNIEnv * env,
struct PacketData *packet_data) {
int got_frame_ptr;
struct Player *player = decoder_data->player;
int stream_no = decoder_data->stream_no;
AVCodecContext * ctx = player->input_codec_ctxs[stream_no];
AVFrame * frame = player->input_frames[stream_no];
AVStream * stream = player->input_streams[stream_no];
int interrupt_ret;
int to_write;
int err = 0;
AVFrame *rgb_frame = player->rgb_frame;
ANativeWindow_Buffer buffer;
ANativeWindow * window;
#ifdef MEASURE_TIME
struct timespec timespec1, timespec2, diff;
#endif // MEASURE_TIME
LOGI(10, "player_decode_video decoding");
int frameFinished;
#ifdef MEASURE_TIME
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec1);
#endif // MEASURE_TIME
int ret = avcodec_decode_video2(ctx, frame, &frameFinished,
packet_data->packet);
#ifdef MEASURE_TIME
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec2);
diff = timespec_diff(timespec1, timespec2);
LOGI(3, "decode_video timediff: %d.%9ld", diff.tv_sec, diff.tv_nsec);
#endif // MEASURE_TIME
if (ret < 0) {
LOGE(1, "player_decode_video Fail decoding video %d\n", ret);
return -ERROR_WHILE_DECODING_VIDEO;
}
if (!frameFinished) {
LOGI(10, "player_decode_video Video frame not finished\n");
return 0;
}
// saving in buffer converted video frame
LOGI(7, "player_decode_video copy wait");
#ifdef MEASURE_TIME
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec1);
#endif // MEASURE_TIME
pthread_mutex_lock(&player->mutex_queue);
window = player->window;
if (window == NULL) {
pthread_mutex_unlock(&player->mutex_queue);
goto skip_frame;
}
ANativeWindow_setBuffersGeometry(window, ctx->width, ctx->height,
WINDOW_FORMAT_RGBA_8888);
if (ANativeWindow_lock(window, &buffer, NULL) != 0) {
pthread_mutex_unlock(&player->mutex_queue);
goto skip_frame;
}
pthread_mutex_unlock(&player->mutex_queue);
int format = buffer.format;
if (format < 0) {
LOGE(1, "Could not get window format")
}
enum PixelFormat out_format;
if (format == WINDOW_FORMAT_RGBA_8888) {
out_format = PIX_FMT_RGBA;
LOGI(6, "Format: WINDOW_FORMAT_RGBA_8888");
} else if (format == WINDOW_FORMAT_RGBX_8888) {
out_format = PIX_FMT_RGB0;
LOGE(1, "Format: WINDOW_FORMAT_RGBX_8888 (not supported)");
} else if (format == WINDOW_FORMAT_RGB_565) {
out_format = PIX_FMT_RGB565;
LOGE(1, "Format: WINDOW_FORMAT_RGB_565 (not supported)");
} else {
LOGE(1, "Unknown window format");
}
avpicture_fill((AVPicture *) rgb_frame, buffer.bits, out_format,
buffer.width, buffer.height);
rgb_frame->data[0] = buffer.bits;
if (format == WINDOW_FORMAT_RGBA_8888) {
rgb_frame->linesize[0] = buffer.stride * 4;
} else {
LOGE(1, "Unknown window format");
}
LOGI(6,
"Buffer: width: %d, height: %d, stride: %d",
buffer.width, buffer.height, buffer.stride);
int i = 0;
#ifdef MEASURE_TIME
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec2);
diff = timespec_diff(timespec1, timespec2);
LOGI(1,
"lockPixels and fillimage timediff: %d.%9ld", diff.tv_sec, diff.tv_nsec);
#endif // MEASURE_TIME
#ifdef MEASURE_TIME
clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec1);
#endif // MEASURE_TIME
LOGI(7, "player_decode_video copying...");
AVFrame * out_frame;
int rescale;
if (ctx->width == buffer.width && ctx->height == buffer.height) {
// This always should be true
out_frame = rgb_frame;
rescale = FALSE;
} else {
out_frame = player->tmp_frame2;
rescale = TRUE;
}
if (ctx->pix_fmt == PIX_FMT_YUV420P) {
__I420ToARGB(frame->data[0], frame->linesize[0], frame->data[2],
frame->linesize[2], frame->data[1], frame->linesize[1],
out_frame->data[0], out_frame->linesize[0], ctx->width,
ctx->height);
} else if (ctx->pix_fmt == PIX_FMT_NV12) {
__NV21ToARGB(frame->data[0], frame->linesize[0], frame->data[1],
frame->linesize[1], out_frame->data[0], out_frame->linesize[0],
ctx->width, ctx->height);
} else {
LOGI(3, "Using slow conversion: %d ", ctx->pix_fmt);
struct SwsContext *sws_context = player->sws_context;
sws_context = sws_getCachedContext(sws_context, ctx->width, ctx->height,
ctx->pix_fmt, ctx->width, ctx->height, out_format,
SWS_FAST_BILINEAR, NULL, NULL, NULL);
player->sws_context = sws_context;
if (sws_context == NULL) {
LOGE(1, "could not initialize conversion context from: %d"
", to :%d\n", ctx->pix_fmt, out_format);
// TODO some error
}
sws_scale(sws_context, (const uint8_t * const *) frame->data,
frame->linesize, 0, ctx->height, out_frame->data,
out_frame->linesize);
}
if (rescale) {
// Never occurs
__ARGBScale(out_frame->data[0], out_frame->linesize[0], ctx->width,
ctx->height, rgb_frame->data[0], rgb_frame->linesize[0],
buffer.width, buffer.height, __kFilterNone);
out_frame = rgb_frame;
}
// TODO: (4ntoine) frame decoded and rescaled, ready to call callback with frame picture from buffer
int bufferSize = buffer.width * buffer.height * 3; // 3 = (r,g,b);
static int bitmapCounter = 0;
if (bitmapCounter < 10) {
saveBuffer(bitmapCounter++, buffer.width, buffer.height, (unsigned char *)out_frame->data, bufferSize);
}
but out_frame is empty and file has header and 0x00 bytes body.
How to get picture buffer data in ffmpeg?
Solved, in short: you should take buffer from ANativeWindow_Buffer - buffer.bits. Pay attention buffer is (rgba) but BMP is usually (rgb) - 3 bytes. To save it as BMP one need to add BMP header and save lines with padding.

How to resize a picture using ffmpeg's sws_scale()?

I wanna resize a picture by using the ffmpeg's func--->sws_scale().
Is there any one knows how to do it?
Do you have the source code for this function?
First you need to create a SwsContext (you need to do this only once) :
struct SwsContext *resize;
resize = sws_getContext(width1, height1, AV_PIX_FMT_YUV420P, width2, height2, PIX_FMT_RGB24, SWS_BICUBIC, NULL, NULL, NULL);
You need two frames for conversion, frame1 is the original frame, you need to explicitly allocate frame2 :
AVFrame* frame1 = avcodec_alloc_frame(); // this is your original frame
AVFrame* frame2 = avcodec_alloc_frame();
int num_bytes = avpicture_get_size(AV_PIX_FMT_RGB24, width2, height2);
uint8_t* frame2_buffer = (uint8_t *)av_malloc(num_bytes*sizeof(uint8_t));
avpicture_fill((AVPicture*)frame2, frame2_buffer, AV_PIX_FMT_RGB24, width2, height2);
You may use this part inside a loop if you need to resize each frame you receive :
// frame1 should be filled by now (eg using avcodec_decode_video)
sws_scale(resize, frame1->data, frame1->linesize, 0, height1, frame2->data, frame2->linesize);
Note that I also changed pixel format, but you can use the same pixel format for both frames
Runnable example in FFmpeg 2.8
Basically using arash's method, but runnable so you can try it out.
Generate one short video procedurally, and then convert it to 3 different sizes.
ffmpeg_encoder_init_frame and ffmpeg_encoder_scale are the key methods.
Source:
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
static AVCodecContext *c = NULL;
static AVFrame *frame;
static AVFrame *frame2;
static AVPacket pkt;
static FILE *file;
static struct SwsContext *sws_context = NULL;
static void ffmpeg_encoder_init_frame(AVFrame **framep, int width, int height) {
int ret;
AVFrame *frame;
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = c->pix_fmt;
frame->width = width;
frame->height = height;
ret = av_image_alloc(frame->data, frame->linesize, frame->width, frame->height, frame->format, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate raw picture buffer\n");
exit(1);
}
*framep = frame;
}
static void ffmpeg_encoder_scale(uint8_t *rgb) {
sws_context = sws_getCachedContext(sws_context,
frame->width, frame->height, AV_PIX_FMT_YUV420P,
frame2->width, frame2->height, AV_PIX_FMT_YUV420P,
SWS_BICUBIC, NULL, NULL, NULL);
sws_scale(sws_context, (const uint8_t * const *)frame->data, frame->linesize, 0,
frame->height, frame2->data, frame2->linesize);
}
static void ffmpeg_encoder_set_frame_yuv_from_rgb(uint8_t *rgb) {
const int in_linesize[1] = { 3 * frame->width };
sws_context = sws_getCachedContext(sws_context,
frame->width, frame->height, AV_PIX_FMT_RGB24,
frame->width, frame->height, AV_PIX_FMT_YUV420P,
0, NULL, NULL, NULL);
sws_scale(sws_context, (const uint8_t * const *)&rgb, in_linesize, 0,
frame->height, frame->data, frame->linesize);
}
void generate_rgb(int width, int height, int pts, uint8_t **rgbp) {
int x, y, cur;
uint8_t *rgb = *rgbp;
rgb = realloc(rgb, 3 * sizeof(uint8_t) * height * width);
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) {
cur = 3 * (y * width + x);
rgb[cur + 0] = 0;
rgb[cur + 1] = 0;
rgb[cur + 2] = 0;
if ((frame->pts / 25) % 2 == 0) {
if (y < height / 2) {
if (x < width / 2) {
/* Black. */
} else {
rgb[cur + 0] = 255;
}
} else {
if (x < width / 2) {
rgb[cur + 1] = 255;
} else {
rgb[cur + 2] = 255;
}
}
} else {
if (y < height / 2) {
rgb[cur + 0] = 255;
if (x < width / 2) {
rgb[cur + 1] = 255;
} else {
rgb[cur + 2] = 255;
}
} else {
if (x < width / 2) {
rgb[cur + 1] = 255;
rgb[cur + 2] = 255;
} else {
rgb[cur + 0] = 255;
rgb[cur + 1] = 255;
rgb[cur + 2] = 255;
}
}
}
}
}
*rgbp = rgb;
}
void ffmpeg_encoder_start(const char *filename, int codec_id, int fps, int width, int height, float factor) {
AVCodec *codec;
int ret;
int width2 = width * factor;
int height2 = height * factor;
avcodec_register_all();
codec = avcodec_find_encoder(codec_id);
if (!codec) {
fprintf(stderr, "Codec not found\n");
exit(1);
}
c = avcodec_alloc_context3(codec);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
c->bit_rate = 400000;
c->width = width2;
c->height = height2;
c->time_base.num = 1;
c->time_base.den = fps;
c->gop_size = 10;
c->max_b_frames = 1;
c->pix_fmt = AV_PIX_FMT_YUV420P;
if (codec_id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
file = fopen(filename, "wb");
if (!file) {
fprintf(stderr, "Could not open %s\n", filename);
exit(1);
}
ffmpeg_encoder_init_frame(&frame, width, height);
ffmpeg_encoder_init_frame(&frame2, width2, height2);
}
void ffmpeg_encoder_finish(void) {
uint8_t endcode[] = { 0, 0, 1, 0xb7 };
int got_output, ret;
do {
fflush(stdout);
ret = avcodec_encode_video2(c, &pkt, NULL, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
fwrite(pkt.data, 1, pkt.size, file);
av_packet_unref(&pkt);
}
} while (got_output);
fwrite(endcode, 1, sizeof(endcode), file);
fclose(file);
avcodec_close(c);
av_free(c);
av_freep(&frame->data[0]);
av_frame_free(&frame);
av_freep(&frame2->data[0]);
av_frame_free(&frame2);
}
void ffmpeg_encoder_encode_frame(uint8_t *rgb) {
int ret, got_output;
ffmpeg_encoder_set_frame_yuv_from_rgb(rgb);
ffmpeg_encoder_scale(rgb);
frame2->pts = frame->pts;
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
ret = avcodec_encode_video2(c, &pkt, frame2, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
fwrite(pkt.data, 1, pkt.size, file);
av_packet_unref(&pkt);
}
}
static void encode_example(float factor) {
char filename[255];
int pts;
int width = 320;
int height = 240;
uint8_t *rgb = NULL;
snprintf(filename, 255, "tmp." __FILE__ ".%.2f.h264", factor);
ffmpeg_encoder_start(filename, AV_CODEC_ID_H264, 25, width, height, factor);
for (pts = 0; pts < 100; pts++) {
frame->pts = pts;
generate_rgb(width, height, pts, &rgb);
ffmpeg_encoder_encode_frame(rgb);
}
ffmpeg_encoder_finish();
free(rgb);
}
int main(void) {
encode_example(0.5);
encode_example(1.0);
encode_example(2.0);
return EXIT_SUCCESS;
}
Run with:
gcc main.c -lavformat -lavcodec -lswresample -lswscale -lavutil -lx264
./a.out
ffplay tmp.main.c.0.50.h264
ffplay tmp.main.c.1.00.h264
ffplay tmp.main.c.2.00.h264
Tested on ffmpeg 3.4.11, Ubuntu 16.04. Source on GitHub. Adapted from doc/examples/decoding_encoding.c.

Resources