Writing to ffpmeg stdin freezes program - go

I'm trying to convert a file in memory using ffmpeg to another format by using stdin and stdout, but everytime I try to write to stdin, of my ffmpeg command, it just freezes there.
package main
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
)
func test(bytes []byte) ([]byte, error) {
cmd := exec.Command(
"ffmpeg",
"-i", "pipe:0", // read from stdin
"-vcodec", "copy",
"-acodec", "copy",
"-f", "matroska",
"pipe:1",
)
in, err := cmd.StdinPipe()
if err != nil {
panic(err)
}
out, err := cmd.StdoutPipe()
if err != nil {
panic(err)
}
fmt.Println("starting")
err = cmd.Start()
if err != nil {
panic(err)
}
fmt.Println("writing")
w := bufio.NewWriter(in)
_, err = w.Write(bytes)
if err != nil {
panic(err)
}
err = w.Flush()
if err != nil {
panic(err)
}
err = in.Close()
if err != nil {
panic(err)
}
fmt.Println("reading")
outBytes, err := io.ReadAll(out)
if err != nil {
panic(err)
}
fmt.Println("waiting")
err = cmd.Wait()
if err != nil {
panic(err)
}
return outBytes, nil
}
func main() {
dat, err := os.ReadFile("speech.mp4")
if err != nil {
panic(err)
}
out, err := test(dat)
if err != nil {
panic(err)
}
err = os.WriteFile("test.m4v", out, 0644)
if err != nil {
panic(err)
}
}
It prints
starting
writing
and gets stuck there. I tried similar code with grep, and the everything worked fine, so this seems to be some ffmpeg specific problem.
I tried running
cat speech.mp4 | ffmpeg -i pipe:0 -vcodec copy -acodec copy -f matroska pipe:1 | cat > test.mkv
and that works fine, so it's not an ffmpeg problem, but some problem with how I'm piping/reading/writing my data.
My speech.mp4 file is around 2MB.

So the secret lied in reading stdout as you dumped the bytes into stdin, since writing to the pipe blocks. Thanks #JimB for helping me figure this out.
You just have to read as you write:
cmd := exec.Command(
"ffmpeg",
"-i", "pipe:0", // read from stdin
"-vcodec", "copy",
"-acodec", "copy",
"-f", "matroska",
"pipe:1",
)
out, err := cmd.StdoutPipe()
if err != nil {
panic(err)
}
in, err := cmd.StdinPipe()
writer := bufio.NewWriter(in)
if err != nil {
panic(err)
}
fmt.Println("starting")
err = cmd.Start()
if err != nil {
panic(err)
}
go func() {
defer writer.Flush()
defer in.Close()
fmt.Println("writing")
_, err = writer.Write(bytes)
if err != nil {
panic(err)
}
}()
var outBytes []byte
defer out.Close()
fmt.Println("reading")
outBytes, err = io.ReadAll(out)
if err != nil {
panic(err)
}
fmt.Println("waiting")
err = cmd.Wait()
if err != nil {
panic(err)
}
return outBytes, nil

Related

Saving a continuous stream of images from ffmpeg image2pipe

I am trying to save a sequence/continuous images from ffmpeg image2pipe in go. The problem with the code below that it does only save the first image in the stream due to the blocking nature of io.Copy since it waits for the reader or the writer to close.
package main
import (
"fmt"
"io"
"log"
"os"
"os/exec"
"strconv"
"time"
)
//Trying to get png from stdout pipe
func main() {
fmt.Println("Running the camera stream")
ffmpegCmd := exec.Command("ffmpeg", "-loglevel", "quiet", "-y", "-rtsp_transport", "tcp", "-i", "rtsp://admin:123456#192.168.1.41:554/h264Preview_01_main", "-r", "1", "-f", "image2pipe", "pipe:1")
ffmpegOut, err := ffmpegCmd.StdoutPipe()
if err != nil {
return
}
err = ffmpegCmd.Start()
if err != nil {
log.Fatal(err)
}
count := 0
for {
count++
t := time.Now()
fmt.Println("writing image" + strconv.Itoa(count))
filepath := "image-" + strconv.Itoa(count) + "-" + t.Format("20060102150405.png")
out, err := os.Create(filepath)
if err != nil {
log.Fatal(err)
}
defer out.Close()
_, err = io.Copy(out, ffmpegOut)
if err != nil {
log.Fatalf("unable to copy to file: %s", err.Error())
}
}
if err := ffmpegCmd.Wait(); err != nil {
log.Fatal("Error while waiting:", err)
}
}
I implemented my own save and copy function based on the io.Copy code https://golang.org/src/io/io.go
func copyAndSave(w io.Writer, r io.Reader) error {
buf := make([]byte, 1024, 1024)
for {
n, err := r.Read(buf[:])
if n == 0 {
}
if n > 0 {
d := buf[:n]
_, err := w.Write(d)
if err != nil {
return err
}
}
if err != nil {
return err
}
}
return nil
}
then I updated the for loop in my main function to the below block but still I am only getting the first image in the sequence. due to r.Read(buf[:]) is being a blocking call.
for {
count++
t := time.Now()
fmt.Println("writing image" + strconv.Itoa(count))
filepath := "image-" + strconv.Itoa(count) + "-" + t.Format("20060102150405.png")
out, err := os.Create(filepath)
if err != nil {
log.Fatal(err)
}
defer out.Close()
err = copyAndSave(out, ffmpegOut)
if err != nil {
if err == io.EOF {
break
}
log.Fatalf("unable to copy to file: %s", err.Error())
break
}
}

exec ffmpeg stdout pipe stalling

I've been trying to get the exec stdoutpipe from ffmpeg and write it into a different file. However, it stalls and doesn't finish executing the command.
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
)
func stdinfill(stdin io.WriteCloser) {
fi, err := ioutil.ReadFile("music.ogg")
if err != nil {
log.Fatal(err)
}
io.Copy(stdin, bytes.NewReader(fi))
}
func main() {
runcommand()
}
func runcommand() {
cmd := exec.Command("ffmpeg", "-i", "pipe:0", "-f", "mp3", "pipe:1")
stdin, err := cmd.StdinPipe()
if err != nil {
log.Fatal(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
cmd.Stderr = os.Stderr
err = cmd.Start()
if err != nil {
log.Fatal(err)
}
stdinfill(stdin)
fo, err := os.Create("output.mp3")
if err != nil {
log.Fatal(err)
}
io.Copy(fo, stdout)
defer fo.Close()
err = cmd.Wait()
if err != nil {
log.Fatal(err)
}
}
does anyone have any ideas? It starts running ffmpeg but just stalls.
Running stdinfill in a goroutine fixed the issue.

Pipe to exec'ed process

My Go application outputs some amounts of text data and I need to pipe it to some external command (e.g. less). I haven't find any way to pipe this data to syscall.Exec'ed process.
As a workaround I write that text data to a temporary file and then use that file as an argument to less:
package main
import (
"io/ioutil"
"log"
"os"
"os/exec"
"syscall"
)
func main() {
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
log.Fatal(err)
}
defer os.Remove(tmpfile.Name()) // Never going to happen!
if _, err := tmpfile.Write(content); err != nil {
log.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
log.Fatal(err)
}
binary, err := exec.LookPath("less")
if err != nil {
log.Fatal(err)
}
args := []string{"less", tmpfile.Name()}
if err := syscall.Exec(binary, args, os.Environ()); err != nil {
log.Fatal(err)
}
}
It works but leaves a temporary file on a file system, because syscall.Exec replaces the current Go process with another (less) one and deferred os.Remove won't run. Such behaviour is not desirable.
Is there any way to pipe some data to an external process without leaving any artefacts?
You should be using os/exec to build an exec.Cmd to execute, then you could supply any io.Reader you want as the stdin for the command.
From the example in the documentation:
cmd := exec.Command("tr", "a-z", "A-Z")
cmd.Stdin = strings.NewReader("some input")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
fmt.Printf("in all caps: %q\n", out.String())
If you want to write directly to the command's stdin, then you call cmd.StdInPipe to get an io.WriteCloser you can write to.
If you really need to exec the process in place of your current one, you can simply remove the file before exec'ing, and provide that file descriptor as stdin for the program.
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
log.Fatal(err)
}
os.Remove(tmpfile.Name())
if _, err := tmpfile.Write(content); err != nil {
log.Fatal(err)
}
tmpfile.Seek(0, 0)
err = syscall.Dup2(int(tmpfile.Fd()), syscall.Stdin)
if err != nil {
log.Fatal(err)
}
binary, err := exec.LookPath("less")
if err != nil {
log.Fatal(err)
}
args := []string{"less"}
if err := syscall.Exec(binary, args, os.Environ()); err != nil {
log.Fatal(err)
}

golang scp file using crypto/ssh

I'm trying to download a remote file over ssh
The following approach works fine on shell
ssh hostname "tar cz /opt/local/folder" > folder.tar.gz
However the same approach on golang giving some difference in output artifact size. For example the same folders with pure shell produce artifact gz file 179B and same with go script 178B.
I assume that something has been missed from io.Reader or session got closed earlier. Kindly ask you guys to help.
Here is the example of my script:
func executeCmd(cmd, hostname string, config *ssh.ClientConfig, path string) error {
conn, _ := ssh.Dial("tcp", hostname+":22", config)
session, err := conn.NewSession()
if err != nil {
panic("Failed to create session: " + err.Error())
}
r, _ := session.StdoutPipe()
scanner := bufio.NewScanner(r)
go func() {
defer session.Close()
name := fmt.Sprintf("%s/backup_folder_%v.tar.gz", path, time.Now().Unix())
file, err := os.OpenFile(name, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
panic(err)
}
defer file.Close()
for scanner.Scan() {
fmt.Println(scanner.Bytes())
if err := scanner.Err(); err != nil {
fmt.Println(err)
}
if _, err = file.Write(scanner.Bytes()); err != nil {
log.Fatal(err)
}
}
}()
if err := session.Run(cmd); err != nil {
fmt.Println(err.Error())
panic("Failed to run: " + err.Error())
}
return nil
}
Thanks!
bufio.Scanner is for newline delimited text. According to the documentation, the scanner will remove the newline characters, stripping any 10s out of your binary file.
You don't need a goroutine to do the copy, because you can use session.Start to start the process asynchronously.
You probably don't need to use bufio either. You should be using io.Copy to copy the file, which has an internal buffer already on top of any buffering already done in the ssh client itself. If an additional buffer is needed for performance, wrap the session output in a bufio.Reader
Finally, you return an error value, so use it rather than panic'ing on regular error conditions.
conn, err := ssh.Dial("tcp", hostname+":22", config)
if err != nil {
return err
}
session, err := conn.NewSession()
if err != nil {
return err
}
defer session.Close()
r, err := session.StdoutPipe()
if err != nil {
return err
}
name := fmt.Sprintf("%s/backup_folder_%v.tar.gz", path, time.Now().Unix())
file, err := os.OpenFile(name, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return err
}
defer file.Close()
if err := session.Start(cmd); err != nil {
return err
}
n, err := io.Copy(file, r)
if err != nil {
return err
}
if err := session.Wait(); err != nil {
return err
}
return nil
You can try doing something like this:
r, _ := session.StdoutPipe()
reader := bufio.NewReader(r)
go func() {
defer session.Close()
// open file etc
// 10 is the number of bytes you'd like to copy in one write operation
p := make([]byte, 10)
for {
n, err := reader.Read(p)
if err == io.EOF {
break
}
if err != nil {
log.Fatal("err", err)
}
if _, err = file.Write(p[:n]); err != nil {
log.Fatal(err)
}
}
}()
Make sure your goroutines are synchronized properly so output is completeky written to the file.

Strange behaviour of golang exec.Command program

I have such code:
func main() {
s := "foobar"
cmd := exec.Command("wc", "-l")
stdin, err := cmd.StdinPipe()
if err != nil {
log.Panic(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Panic(err)
}
err = cmd.Start()
if err != nil {
log.Panic(err)
}
io.Copy(stdin, bytes.NewBufferString(s))
stdin.Close()
io.Copy(os.Stdout, stdout)
err = cmd.Wait()
if err != nil {
log.Panic(err)
}
}
and its output is:
0
But when I will do simple modification:
func main() {
runWcFromStdinWorks("aaa\n")
runWcFromStdinWorks("bbb\n")
}
func runWcFromStdinWorks(s string) {
cmd := exec.Command("wc", "-l")
stdin, err := cmd.StdinPipe()
if err != nil {
log.Panic(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Panic(err)
}
err = cmd.Start()
if err != nil {
log.Panic(err)
}
io.Copy(stdin, bytes.NewBufferString(s))
stdin.Close()
io.Copy(os.Stdout, stdout)
err = cmd.Wait()
if err != nil {
log.Panic(err)
}
}
It works, but why? Its just calling method why first version is not working?
The string s in the first example does not have a new line, which causes wc -l to return 0. You can see this behavior by doing:
$ echo -n hello | wc -l
0

Resources