How to convert VP8 interframe into image with Pion/Webrtc? - go

Question

I have tried snapshot example to convert VP8 video stream to jpeg in Pion/Webrtc examples.
It works well, but interframes are not included. An error message displayed when I was fixing it.
vp8: Golden / AltRef frames are not implemented.
This error comes from golang's vp8 package.
The modified code is shown below:
if !videoKeyframe {
decoder.Init(bytes.NewReader(sample.Data), len(sample.Data))
_, err := decoder.DecodeFrameHeader()
if err != nil {
fmt.Printf("Header Error: %s\n", err)
continue
}
_, err2 := decoder.DecodeFrame()
if err2 != nil {
fmt.Printf("DecodeFrame Error: %s\n", err2)
continue
}
}
I also tried to modify gstreamer-receive example. But it's hard for me to do in C.
Is there anyway to convert interframe to jpeg easily ?

Answer 1

Finally, I solved with libvpx-go
I modified code as shown below.
package main
import (
"bytes"
"fmt"
"image/jpeg"
// "log"
"os"
"github.com/pion/rtp"
"github.com/pion/rtp/codecs"
"github.com/pion/webrtc/v3/pkg/media/samplebuilder"
"github.com/xlab/libvpx-go/vpx"
)
// type Frame struct {
// *image.RGBA
// Timecode time.Duration
// IsKeyframe bool
// }
type VDecoder struct {
enabled bool
src <-chan *rtp.Packet
ctx *vpx.CodecCtx
iface *vpx.CodecIface
}
type VCodec string
const (
CodecVP8 VCodec = "V_VP8"
CodecVP9 VCodec = "V_VP9"
CodecVP10 VCodec = "V_VP10"
)
func NewVDecoder(codec VCodec, src <-chan *rtp.Packet) *VDecoder {
dec := &VDecoder{
src: src,
ctx: vpx.NewCodecCtx(),
}
switch codec {
case CodecVP8:
dec.iface = vpx.DecoderIfaceVP8()
case CodecVP9:
dec.iface = vpx.DecoderIfaceVP9()
default: // others are currently disabled
log.Println("[WARN] unsupported VPX codec:", codec)
return dec
}
err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
if err != nil {
log.Println("[WARN]", err)
return dec
}
dec.enabled = true
return dec
}
func (v *VDecoder) Save(savePath string) { //, out chan<- Frame
// defer close(out)
i := 0
sampleBuilder := samplebuilder.New(20000, &codecs.VP8Packet{}, 90000)
for pkt := range v.src {
sampleBuilder.Push(pkt)
// Use SampleBuilder to generate full picture from many RTP Packets
sample := sampleBuilder.Pop()
if sample == nil {
continue
}
if !v.enabled {
continue
}
dataSize := uint32(len(sample.Data))
err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample.Data), dataSize, nil, 0))
if err != nil {
log.Println("[WARN]", err)
continue
}
var iter vpx.CodecIter
img := vpx.CodecGetFrame(v.ctx, &iter)
if img != nil {
img.Deref()
// out <- Frame{
// RGBA: img.ImageRGBA(),
// Timecode: time.Duration(pkt.Timestamp),
// }
i++
buffer := new(bytes.Buffer)
if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
// panic(err)
fmt.Printf("jpeg Encode Error: %s\r\n", err)
}
fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, i, ".jpg"))
if err != nil {
fmt.Printf("image create Error: %s\r\n", err)
//panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
if _, err := fo.Write(buffer.Bytes()); err != nil {
fmt.Printf("image write Error: %s\r\n", err)
//panic(err)
}
fo.Close()
}
}
}
Just add 2 lines of code to save sequential images
vd := NewVDecoder(CodecVP8, rtpChan)
go vd.Save("/ramdisk/image_seq/xxx")
Enjoy!
P.S. don't forget to install libvpx-go 's dependencies before running go get and make sure libvpx 's version is 1.8

Answer 2

If you need to convert interframe to jpeg the only way would be to decode every frame from your stream. I don't think there is any other way as interframes depend on previous frames.

Answer 3

Sync Solution
This is an alternative version of the same solution of CK So
It is based on libvpx-go but dose not use channels.
package main
import (
"bytes"
"fmt"
"image/jpeg"
"log"
// "log"
"os"
"github.com/xlab/libvpx-go/vpx"
)
// type Frame struct {
// *image.RGBA
// Timecode time.Duration
// IsKeyframe bool
// }
type VDecoderSync struct {
enabled bool
//src <-chan []byte this blocked the thread even if fed.
ctx *vpx.CodecCtx
iface *vpx.CodecIface
i int
}
type VCodecSync string
const (
CodecVP8Sync VCodecSync = "V_VP8"
CodecVP9Sync VCodecSync = "V_VP9"
CodecVP10Sync VCodecSync = "V_VP10"
)
func NewVDecoderSync(codec VCodecSync) *VDecoderSync {
dec := &VDecoderSync{
ctx: vpx.NewCodecCtx(),
}
switch codec {
case CodecVP8Sync:
dec.iface = vpx.DecoderIfaceVP8()
case CodecVP9Sync:
dec.iface = vpx.DecoderIfaceVP9()
default: // others are currently disabled
log.Println("[WARN] unsupported VPX codec:", codec)
return dec
}
err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
if err != nil {
log.Println("[WARN]", err)
return dec
}
dec.enabled = true
return dec
}
func (v *VDecoderSync) EncodeToJson(savePath string, pkt []byte, print_to_file bool) []byte {
sample := pkt
dataSize := uint32(len(sample))
err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample), dataSize, nil, 0))
if err != nil {
log.Println("[WARN]", err)
return nil
}
var iter vpx.CodecIter
img := vpx.CodecGetFrame(v.ctx, &iter)
if img != nil {
img.Deref()
// out <- Frame{
// RGBA: img.ImageRGBA(),
// Timecode: time.Duration(pkt.Timestamp),
// }
buffer := new(bytes.Buffer)
if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
// panic(err)
fmt.Printf("jpeg Encode Error: %s\r\n", err)
}
bytes_to_send := buffer.Bytes()
if print_to_file {
v.i++
v.i = v.i % 30
fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, v.i, ".jpg"))
if err != nil {
fmt.Printf("image create Error: %s\r\n", err)
//panic(err)
}
// close fo on exit and check for its returned error
defer func() {
if err := fo.Close(); err != nil {
panic(err)
}
}()
if _, err := fo.Write(bytes_to_send); err != nil {
fmt.Printf("image write Error: %s\r\n", err)
//panic(err)
}
}
return bytes_to_send
}
return nil
}