I'm writing a kind of skype application in which I first encode the video with x265 encoder. Below you can see the settings of the encoder and the line to encode the video and get the nal_units.
x265_param_default_preset(param, "ultrafast", "zerolatency");
x265_param_parse(param, "fps", "30");
x265_param_parse(param, "input-res", "160x120"); //wxh
x265_param_parse(param, "bframes", "3");
x265_param_parse(param, "rc-lookahead", "5");
x265_param_parse(param, "repeat-headers", "1");
x265_nal *pp_nal=NULL;
uint32_t pi_nal=0;
int encoded = x265_encoder_encode(encoder, &pp_nal, &pi_nal, pic_in, pic_recon);
After this I send the pp_nal and pi_nal to the client and then I would like to decode these nal_units to yuv with ffmpeg. I initialize my decoder:
AVCodec *codec;
AVCodecContext *av_codec_context = NULL;
avcodec_register_all();
int frame_count;
FILE *f;
AVFrame *frame;
AVPacket avpkt;
av_init_packet(&avpkt);
codec = avcodec_find_decoder(AV_CODEC_ID_H265);
if (!codec) {
fprintf(stderr, "Codec not found\n");
exit(1);
}
av_codec_context = avcodec_alloc_context3(codec);
if (!av_codec_context) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
av_codec_context->width = 160;
av_codec_context->height = 120;
av_codec_context->extradata = NULL;
av_codec_context->pix_fmt = PIX_FMT_YUV420P;
/* open it */
if (avcodec_open2(av_codec_context, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
AVFrame *av_frame_ = av_frame_alloc();
frame_count = 0;
int got_frame;
And then for every nal packet I execute the following:
AVPacket av_packet;
av_new_packet(&av_packet, pp_nal->sizeBytes);
av_packet.data = (uint8_t *) pp_nal->payload;
av_packet.size = pp_nal->sizeBytes;
avcodec_decode_video2(av_codec_context, av_frame_, &got_frame, &av_packet);
But this doesn't work and gives the followig error 'Error parsing nal unit'. What am I doing wrong?
Related
I have a binary file with raw h264 data which is arranged like that
NAL(SPS), NAL(PPS), NAL(Frame), NAL(SPS), NAL(PPS)....
and i want to mux it (without encode) into a mp4 container.
The muxing.c in the ffmpeg example do the encoding of yuv data, but it is different from my case, and i have no ideas how to change the example to do what i want to do...
I knew the commaned ffmpeg -i h264file -c copy h264.mp4 can do what i want to do, but i have to do it in my program, so i need to know how to use the ffmpeg api to do the same thing, but so far, i cannot find any simple example to do it. Is there anyone has hint on how to do it?? Thanks
Updated, i have write the code as below from the reference, it seems can create the mp4 but the time is not correct, it lost the frame rate information and the time information, it play very fast.
av_register_all();
int ret;
AVDictionary *opt = NULL;
//bool is264 = true;
const char * inputFileName = "input.264";
const char * outputFileName = "output.mp4";
AVFormatContext *ic = avformat_alloc_context();
if((ret = avformat_open_input(&ic, inputFileName, NULL, NULL)) < 0)
return -1;//
// Get format info (retrieve stream information)
if ((ret = avformat_find_stream_info(ic, NULL)) < 0)
return ret; // Couldn't find stream information
for (int i = 0; i < ic->nb_streams; i++)
{
AVStream *stream;
AVCodecContext *codec_ctx;
stream = ic->streams[i];
codec_ctx = stream->codec;
/* Reencode video & audio and remux subtitles etc. */
if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO
|| codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
/* Open decoder */
ret = avcodec_open2(codec_ctx, avcodec_find_decoder(codec_ctx->codec_id), NULL);
if (ret < 0) {
//av_log(NULL, AV_LOG_ERROR, "Failed to open decoder for stream #%u\n", i);
return ret;
}
}
}
// Dump information about file onto standard error
av_dump_format(ic, 0, inputFileName, 0);
AVFormatContext *oc;
avformat_alloc_output_context2(&oc, NULL, NULL, outputFileName);
if (!oc) {
//printf("Could not deduce output format from file extension: using MPEG.\n");
//avformat_alloc_output_context2(&oc, NULL, "mpeg", outputFileName);
return -1;
}
AVStream *ist = ic->streams[0];
AVCodec *out_vid_codec = avcodec_find_encoder(oc->oformat->video_codec);
if (NULL == out_vid_codec)
return -1; // Couldn't find video encoder
AVStream *out_vid_strm = avformat_new_stream(oc, out_vid_codec);
if (NULL == out_vid_strm)
return -1; // Couldn't output video stream
ret = avcodec_copy_context(out_vid_strm->codec, ist->codec);
if (ret < 0)
return ret; // Failed to copy context
ret = avio_open(&oc->pb, outputFileName, AVIO_FLAG_WRITE);
ret = avformat_write_header(oc, NULL);
AVPacket pkt;
while(1)
{
AVStream *in_stream, *out_stream;
ret = av_read_frame(ic, &pkt);
if (ret < 0)
break;
pkt.stream_index = 0;
in_stream = ic->streams[pkt.stream_index];
out_stream = oc->streams[pkt.stream_index];
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
//log_packet(ofmt_ctx, &pkt, "out");
ret = av_interleaved_write_frame(oc, &pkt);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&pkt);
}
av_write_trailer(oc);
I have a problem with decoding of gray images encoded with FFV1 codec.
I successfully encode 16 bit gray image (with avcodec_receive_packet(...) function) and save AvPacket data to file. Then I read this data from file and try to decode (with avcodec_decode_video2 or avcodec_send_packet/avcodec_receive_frame) with no success:
when I try to decode packet with avcodec_decode_video2 function I get an error "Access violation occurred, unable to write location 0x0000000000000000".
when I try to decode packet with avcodec_send_packet/avcodec_receive_frame functions I get an error "chroma shift parameters 7 0 are invalid".
I compared packets after encoding and before decoding and all fields and values seems to be the same. I even try to decode packet just after avcodec_receive_packet (encoding function), however with the same error.
I use the 4.0 version of ffmpeg and the program is based on decode_video.c and encode_video.c examples.
When I use containers (eg. avi) to support read/write encoded images from file (based on demuxing_decoding.c and muxing.c examples) I successfully encode and decode frames with FFV1. However I cannot use containers, because I want to encode frames with different resolutions and mix few video sources together. Additionally the compression level is significantly lower (falls from 2.9 to 2.2) for few hundred of images, what is also very surprising.
So my question is how to correctly save/read (from binary file not container) and prepare AVPacker for decoding with FFV1.
Any help is greatly appreciated.
The decoding code:
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/opt.h>
#include <libavutil/imgutils.h>
}
#pragma warning(disable: 4996)
#define INBUF_SIZE 4096
#define FF_INPUT_BUFFER_PADDING_SIZE 64
uint8_t endcode[4];
AVCodecContext *c, c2;
AVCodec *codec;
int i, ret, x, y;
AVFrame *frame;
AVPacket *pkt, *pkt_temp;
FILE *encodedVideoFile;
AVDictionary *opts = NULL;
uint8_t *video_dst_data[4] = { NULL };
int video_dst_linesize[4];
int imageSize;
uint8_t inbuf[INBUF_SIZE + AV_INPUT_BUFFER_PADDING_SIZE];
/* flush the encoder */
frame = NULL;
encode();
/* add sequence end code to have a real MPEG file */
//fwrite(endcode, 1, sizeof(endcode), encodedVideoFile);
fclose(encodedVideoFile);
avcodec_free_context(&c);
av_frame_free(&frame);
av_packet_free(&pkt);
}
void initDecoding(const char *filename)
{
/* set end of buffer to 0 (this ensures that no overreading happens for damaged MPEG streams) */
memset(inbuf + INBUF_SIZE, 0, AV_INPUT_BUFFER_PADDING_SIZE);
/* find the MPEG-1 video decoder */
codec = avcodec_find_decoder(AV_CODEC_ID_FFV1);
if (!codec) {
fprintf(stderr, "Codec not found\n");
exit(1);
}
c = avcodec_alloc_context3(codec);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
/* resolution must be a multiple of two */
c->width = 1280;
c->height = 484;
/* frames per second */
c->time_base.den = 1;
c->time_base.num = 10;
c->bits_per_raw_sample = 16;
c->framerate.den = 10;
c->framerate.num = 1;
c->pix_fmt = AV_PIX_FMT_GRAY16;
//Version of FFV1 codec
c->level = 3;
/* Init the decoders, with or without reference counting */
av_dict_set(&opts, "refcounted_frames", 0 ? "1" : "0", 0);
if ((ret = avcodec_open2(c, codec, &opts)) < 0) {
return;
}
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
ret = av_image_alloc(video_dst_data, video_dst_linesize,
c->width, c->height, c->pix_fmt, 4);
if (ret < 0) {
fprintf(stderr, "Could not allocate raw video buffer\n");
}
encodedVideoFile = fopen(filename, "rb");
if (!encodedVideoFile) {
fprintf(stderr, "Could not open %s\n", filename);
exit(1);
}
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = c->pix_fmt;
frame->width = c->width;
frame->height = c->height;
ret = av_frame_get_buffer(frame, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate the video frame data\n");
exit(1);
}
/* make sure the frame data is writable */
ret = av_frame_make_writable(frame);
if (ret < 0)
exit(1);
}
void closeDecoding()
{
fclose(encodedVideoFile);
av_parser_close(parser);
avcodec_free_context(&c);
av_frame_free(&frame);
av_packet_free(&pkt);
}
void decodePacket()
{
size_t data_size;
int *got_frame = 0;
read_packt_from_file(pkt, encodedVideoFile);
ret = av_frame_is_writable(frame);
//First decoding function
/*ret = avcodec_decode_video2(c, frame, got_frame, pkt);
if (ret < 0) {
fprintf(stderr, "Error decoding video frame (%s)\n");
}*/
ret = avcodec_send_packet(c, pkt);
if (ret < 0) {
fprintf(stderr, "Error sending a packet for decoding\n");
exit(1);
}
while (ret >= 0) {
ret = avcodec_receive_frame(c, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
exit(1);
}
printf("saving frame %3d\n", c->frame_number);
fflush(stdout);
}
}
size_t read_packt_from_file(AVPacket *packet, FILE *file)
{
size_t ret = 0;
int size;
uint8_t * data;
//av_packet_from_data
ret = fread(packet, sizeof(AVPacket), 1, file);
size = packet->size;
data = new uint8_t[size];
ret = fread(data, size, 1, file);
av_new_packet(packet, size);
av_packet_from_data(packet, data, size);
return ret;
}
//To write encoded AVPacket
size_t write_packt_to_file(AVPacket *packet, FILE *file)
{
size_t ret = 0;
ret = fwrite(packet, sizeof(AVPacket), 1, file);
ret = fwrite(packet->data, packet->size, 1, file);
if (packet->buf) {
fwrite(packet->buf->data, packet->buf->size, 1, file);
}
fflush(file);
return ret;
}
I'm using libavcodec to encode a stream to h264 and libavformat to store it in an mp4. The resulting container has an invalid header that can be played in VLC, but not any other player.
I've found that using the mp4 container and the "mpeg4" codec produces a valid mp4 file, but using libx265 (HEVC) or the libx264 codec produces invalid mp4s.
I can use ffmpeg -i invalid.mp4 -vcodec copy valid.mp4 and I get a file of almost the exact same size, but in a valid container.
Examples of these files are here: Broken file and
Repaied file [use the download links in the upper right to examine]
I used a hex editor to see the differences in the headers of the two files and the invalid one is 1 byte smaller than the valid one.
The code I'm using to open the container and codec and to write the header is here:
AVOutputFormat *container_format;
AVFormatContext *container_format_context;
AVStream *video_stream;
int ret;
/* allocate the output media context */
avformat_alloc_output_context2(&container_format_context, NULL, NULL, out_file);
if (!container_format_context) {
log(INFO, "Unable to determine container format from filename, exiting\n");
exit(1);
}
else {
log(INFO, "Using container %s\n", container_format_context->oformat->name);
}
if (!container_format_context) {
log(ERROR, "Could not build container format context. Encoding failed.");
exit(1);
}
container_format = container_format_context->oformat;
/* Pull codec based on name */
AVCodec* codec = avcodec_find_encoder_by_name(codec_name);
if (codec == NULL) {
log(ERROR, "Failed to locate codec \"%s\".",
codec_name);
exit(1);
}
/* create stream */
video_stream = NULL;
video_stream = avformat_new_stream(container_format_context, codec);
if (!video_stream) {
log(ERROR, "Could not allocate encoder stream. Cannot continue.\n");
exit(1);
}
video_stream->id = container_format_context->nb_streams - 1;
video_stream->time_base = video_stream->codec->time_base = (AVRational) { 1, 25};
av_dump_format(container_format_context, 0, out_file, 1);
/* Retrieve encoding context */
AVCodecContext* avcodec_context = video_stream->codec;
if (avcodec_context == NULL) {
log(ERROR, "Failed to allocate context for "
"codec \"%s\".", codec_name);
exit(1);
}
/* Init context with encoding parameters */
avcodec_context->bit_rate = bitrate;
avcodec_context->width = width;
avcodec_context->height = height;
avcodec_context->gop_size = 10;
avcodec_context->max_b_frames = 1;
avcodec_context->qmax = 31;
avcodec_context->qmin = 2;
avcodec_context->pix_fmt = AV_PIX_FMT_YUV420P;
av_dump_format(container_format_context, 0, out_file, 1);
/* Open codec for use */
if (avcodec_open2(avcodec_context, codec, NULL) < 0) {
log(ERROR, "Failed to open codec \"%s\".", codec_name);
exit(1);
}
/* Allocate corresponding frame */
AVFrame* frame = av_frame_alloc();
if (frame == NULL) {
exit(1);
}
/* Copy necessary data for frame from avcodec_context */
frame->format = avcodec_context->pix_fmt;
frame->width = avcodec_context->width;
frame->height = avcodec_context->height;
/* Allocate actual backing data for frame */
if (av_image_alloc(frame->data, frame->linesize, frame->width,
frame->height, frame->format, 32) < 0) {
exit(1);
}
/* open the output file, if the container needs it */
if (!(container_format->flags & AVFMT_NOFILE)) {
ret = avio_open(&container_format_context->pb, out_file, AVIO_FLAG_WRITE);
if (ret < 0) {
log(ERROR, "Error occurred while opening output file: %s\n",
av_err2str(ret));
exit(1);
}
}
/* write the stream header, if needed */
ret = avformat_write_header(container_format_context, NULL);
if (ret < 0) {
log(ERROR, "Error occurred while writing output file header: %s\n",
av_err2str(ret));
}
The code to encode a frame is here:
/* Init video packet */
AVPacket packet;
av_init_packet(&packet);
/* Request that encoder allocate data for packet */
packet.data = NULL;
packet.size = 0;
/* Write frame to video */
int got_data;
if (avcodec_encode_video2(avcontext, &packet, frame, &got_data) < 0) {
log(WARNING, "Error encoding frame #%" PRId64,
video_struct->next_pts);
return -1;
}
/* Write corresponding data to file */
if (got_data) {
if (packet.pts != AV_NOPTS_VALUE) {
packet.pts = av_rescale_q(packet.pts, video_struct->output_stream->codec->time_base, video_struct->output_stream->time_base);
}
if (packet.dts != AV_NOPTS_VALUE) {
packet.dts = av_rescale_q(packet.dts, video_struct->output_stream->codec->time_base, video_struct->output_stream->time_base);
}
write_packet(video_struct, &packet, packet.size);
av_packet_unref(&packet);
}
And the code to write the packet to the video stream:
static int write_packet(video_struct* video, void* data, int size) {
int ret;
/* use AVStream is not null, otherwise write to output fd */
AVPacket *pkt = (AVPacket*) data;
pkt->stream_index = video->output_stream->index;
ret = av_interleaved_write_frame(video->container_format_context, pkt);
if (ret != 0) {
return -1;
}
/* Data was written successfully */
return ret;
}
Solved this issue. The problem was that I wasn't assigning global headers to the container if the container required it. While assigning properties like height, width, bit rate and so forth to the avcodec_context, I added
if (container_format_context->oformat->flags & AVFMT_GLOBALHEADER) {
avcodec_context->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
which seems to have fixed the issue.
I would like to ask a question about ffmpeg when i use encoder (x264).
this is my code :
int
FFVideoEncoder::init(AVCodecID codecId, int bitrate, int fps, int gopSize,
int width, int height, AVPixelFormat format) {
release();
const AVCodec *codec = avcodec_find_encoder(codecId);
m_pCodecCtx = avcodec_alloc_context3(codec);
m_pCodecCtx->width = width;
m_pCodecCtx->height = height;
m_pCodecCtx->pix_fmt = format;
m_pCodecCtx->bit_rate = bitrate;
m_pCodecCtx->thread_count = 5;
m_pCodecCtx->max_b_frames = 0;
m_pCodecCtx->gop_size = gopSize;
m_pCodecCtx->time_base.num = 1;
m_pCodecCtx->time_base.den = fps;
//H.264
if (m_pCodecCtx->codec_id == AV_CODEC_ID_H264) {
// av_dict_set(&opts, "preset", "slow", 0);
av_dict_set(&m_pEncoderOpts, "preset", "superfast", 0);
av_dict_set(&m_pEncoderOpts, "tune", "zerolatency", 0);
m_pCodecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER;
m_pCodecCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
int ret = avcodec_open2(m_pCodecCtx, m_pCodecCtx->codec, &m_pEncoderOpts);
if (ret == 0) {
LOGI("open avcodec success!");
} else {
LOGE("open avcodec error!");
return -1;
}
return ret;
}
int FFVideoEncoder::encode(const Frame &inFrame, AVPacket *outPacket) {
AVFrame *frame = av_frame_alloc();
// avpicture_fill((AVPicture *) frame, inFrame.getData(), AV_PIX_FMT_YUV420P, inFrame.getWidth(),
// inFrame.getHeight());
av_image_fill_arrays(frame->data, frame->linesize, inFrame.getData(), m_pCodecCtx->pix_fmt,
inFrame.getWidth(), inFrame.getHeight(), 1);
int ret = 0;
ret = avcodec_send_frame(m_pCodecCtx, frame);
if (ret != 0) {
LOGE("send frame error! %s", av_err2str(ret));
} else {
ret = avcodec_receive_packet(m_pCodecCtx, outPacket);
LOGI("extract data size = %d", m_pCodecCtx->extradata_size);
if (ret != 0) {
LOGE("receive packet error! %s", av_err2str(ret));
}
};
av_frame_free(&frame);
return ret;
}
I expect that the AVPacket will carry the pts and dts about this frame.
but in fact, i only can get encoded frame data and size.
//====================================
except this question, i have another quesiont:
x264 docs say that "tune" opts can be set like film、animation and others. but i only can get a normal video when i set "zerolatency" params. When i set others opts, video's bitrate is very low.
Thanks your answer.
This is for simple example to see if it works:
I believe you should set frame->pts beforehand.
Try this:
Set frame->pts = framecount before sending to ret = avcodec_send_frame(m_pCodecCtx, frame)
Add this framecount as a simple counter of frames you send for encode. Increases each time.
Hope that helps.
I am trying to decode opus using libavcodec. I am able to do it using libopus library alone. But I am trying to acheive same using libavcodec. I am trying to figure it out Why its not working in my case. I have an rtp stream and trying to decode it. The result in decoded packet is same as input. Decoded frame normally contain pcm values instead of that Im receving opus frame that actually I send. Please help me.
av_register_all();
avcodec_register_all();
AVCodec *codec;
AVCodecContext *c = NULL;
AVPacket avpkt;
AVFrame *decoded_frame = NULL;
av_init_packet(&avpkt);
codec = avcodec_find_decoder(AV_CODEC_ID_OPUS);
if (!codec) {
printf("Codec not found\n");
exit(1);
}
c = avcodec_alloc_context3(codec);
if (!c) {
printf("Could not allocate audio codec context\n");
exit(1);
}
/* put sample parameters */
c->sample_rate = 48000;
c->request_sample_fmt = AV_SAMPLE_FMT_FLT;
c->channels = 2;
/* open it */
if (avcodec_open2(c, codec, NULL) < 0) {
printf("Could not open codec\n");
exit(1);
}
AVPacket avpkt;
AVFrame *decoded_frame = NULL;
av_init_packet(&avpkt);
avpkt.data = Buffer; // Buffer is packet data here
avpkt.size = len; // length of the packet
int i, ch;
if (!decoded_frame) {
if (!(decoded_frame = av_frame_alloc())) {
RELAY_SERVER_PRINT("Could not allocate audio frame\n");
exit(1);
}
}
int ret;
int got_frame = 0;
ret = avcodec_decode_audio4(c, decoded_frame, &got_frame, &avpkt);
if (ret < 0) {
fprintf(stderr, "Error decoding audio frame (%s)\n", av_err2str(ret));
return ret;
}
printf("length %i\n", decoded_frame->pkt_size);
I had the same problem. My stream was encoded with 8kHz and ffmpeg is always initializing libopus with 48kHz (hard-coded).
See ffmpeg code snippet:
static av_cold int libopus_decode_init(AVCodecContext *avc)
{
(...)
avc->sample_rate = 48000;
avc->sample_fmt = avc->request_sample_fmt == AV_SAMPLE_FMT_FLT ?
AV_SAMPLE_FMT_FLT : AV_SAMPLE_FMT_S16;
(...)
}
I've replaced that by:
if (avc->sample_rate == 0)
avc->sample_rate = 48000;
and decoding works now. I wonder if this decoder supports dynamic bitrate changes.
The length of the raw frame has to be calculated by:
int frame_size = decoded_frame->nb_samples * av_get_bytes_per_sample(decoded_frame->sample_fmt);