I've been trying to get the exec stdoutpipe from ffmpeg and write it into a different file. However, it stalls and doesn't finish executing the command.
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
)
func stdinfill(stdin io.WriteCloser) {
fi, err := ioutil.ReadFile("music.ogg")
if err != nil {
log.Fatal(err)
}
io.Copy(stdin, bytes.NewReader(fi))
}
func main() {
runcommand()
}
func runcommand() {
cmd := exec.Command("ffmpeg", "-i", "pipe:0", "-f", "mp3", "pipe:1")
stdin, err := cmd.StdinPipe()
if err != nil {
log.Fatal(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
cmd.Stderr = os.Stderr
err = cmd.Start()
if err != nil {
log.Fatal(err)
}
stdinfill(stdin)
fo, err := os.Create("output.mp3")
if err != nil {
log.Fatal(err)
}
io.Copy(fo, stdout)
defer fo.Close()
err = cmd.Wait()
if err != nil {
log.Fatal(err)
}
}
does anyone have any ideas? It starts running ffmpeg but just stalls.
Running stdinfill in a goroutine fixed the issue.
Related
I have a tar.gz file and i need to unpack it using golang.
I've tried libs like "archive/tar" but they gave me error: archive/tar: invalid tar header. Now my idea was to use exec to run tar command and unpack tarball, but it always exits with code 2.
My code:
func unpack(tarName string) error {
path, _ := os.Getwd()
//err := Untar(path+"/"+tarName, path+"/")
fmt.Printf(path + "/" + tarName)
cmd := exec.Command("tar", "-xfv", path+"/"+tarName)
cmd.Stdout = os.Stdout
err := cmd.Run()
return err
}
If you are trying to compress tar.gz file you need first to decompress gzip.
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"log"
"os"
)
func ExtractTarGz(gzipStream io.Reader) {
uncompressedStream, err := gzip.NewReader(gzipStream)
if err != nil {
log.Fatal("ExtractTarGz: NewReader failed")
}
tarReader := tar.NewReader(uncompressedStream)
for true {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("ExtractTarGz: Next() failed: %s", err.Error())
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.Mkdir(header.Name, 0755); err != nil {
log.Fatalf("ExtractTarGz: Mkdir() failed: %s", err.Error())
}
case tar.TypeReg:
outFile, err := os.Create(header.Name)
if err != nil {
log.Fatalf("ExtractTarGz: Create() failed: %s", err.Error())
}
if _, err := io.Copy(outFile, tarReader); err != nil {
log.Fatalf("ExtractTarGz: Copy() failed: %s", err.Error())
}
outFile.Close()
default:
log.Fatalf(
"ExtractTarGz: uknown type: %s in %s",
header.Typeflag,
header.Name)
}
}
}
func main() {
r, err := os.Open("./file.tar.gz")
if err != nil {
fmt.Println("error")
}
ExtractTarGz(r)
}
I'm trying to convert a file in memory using ffmpeg to another format by using stdin and stdout, but everytime I try to write to stdin, of my ffmpeg command, it just freezes there.
package main
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
)
func test(bytes []byte) ([]byte, error) {
cmd := exec.Command(
"ffmpeg",
"-i", "pipe:0", // read from stdin
"-vcodec", "copy",
"-acodec", "copy",
"-f", "matroska",
"pipe:1",
)
in, err := cmd.StdinPipe()
if err != nil {
panic(err)
}
out, err := cmd.StdoutPipe()
if err != nil {
panic(err)
}
fmt.Println("starting")
err = cmd.Start()
if err != nil {
panic(err)
}
fmt.Println("writing")
w := bufio.NewWriter(in)
_, err = w.Write(bytes)
if err != nil {
panic(err)
}
err = w.Flush()
if err != nil {
panic(err)
}
err = in.Close()
if err != nil {
panic(err)
}
fmt.Println("reading")
outBytes, err := io.ReadAll(out)
if err != nil {
panic(err)
}
fmt.Println("waiting")
err = cmd.Wait()
if err != nil {
panic(err)
}
return outBytes, nil
}
func main() {
dat, err := os.ReadFile("speech.mp4")
if err != nil {
panic(err)
}
out, err := test(dat)
if err != nil {
panic(err)
}
err = os.WriteFile("test.m4v", out, 0644)
if err != nil {
panic(err)
}
}
It prints
starting
writing
and gets stuck there. I tried similar code with grep, and the everything worked fine, so this seems to be some ffmpeg specific problem.
I tried running
cat speech.mp4 | ffmpeg -i pipe:0 -vcodec copy -acodec copy -f matroska pipe:1 | cat > test.mkv
and that works fine, so it's not an ffmpeg problem, but some problem with how I'm piping/reading/writing my data.
My speech.mp4 file is around 2MB.
So the secret lied in reading stdout as you dumped the bytes into stdin, since writing to the pipe blocks. Thanks #JimB for helping me figure this out.
You just have to read as you write:
cmd := exec.Command(
"ffmpeg",
"-i", "pipe:0", // read from stdin
"-vcodec", "copy",
"-acodec", "copy",
"-f", "matroska",
"pipe:1",
)
out, err := cmd.StdoutPipe()
if err != nil {
panic(err)
}
in, err := cmd.StdinPipe()
writer := bufio.NewWriter(in)
if err != nil {
panic(err)
}
fmt.Println("starting")
err = cmd.Start()
if err != nil {
panic(err)
}
go func() {
defer writer.Flush()
defer in.Close()
fmt.Println("writing")
_, err = writer.Write(bytes)
if err != nil {
panic(err)
}
}()
var outBytes []byte
defer out.Close()
fmt.Println("reading")
outBytes, err = io.ReadAll(out)
if err != nil {
panic(err)
}
fmt.Println("waiting")
err = cmd.Wait()
if err != nil {
panic(err)
}
return outBytes, nil
I have been testing code using io.Pipe to tar and gunzip files into a tar ball and then unzipping using the tar utility. The follow code passes, however the untaring process keeps getting
error:
tar: Truncated input file (needed 1050624 bytes, only 0 available)
tar: Error exit delayed from previous errors.
This issue is really driving me crazy. It has been two weeks. I really need help debugging.
Thanks.
Development enviroment: go version go1.9 darwin/amd64
package main
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"log"
"os"
"path/filepath"
"testing"
)
func testTarGzipPipe2(t *testing.T) {
src := "/path/to/file/folder"
pr, pw := io.Pipe()
gzipWriter := gzip.NewWriter(pw)
defer gzipWriter.Close()
tarWriter := tar.NewWriter(gzipWriter)
defer tarWriter.Close()
status := make(chan bool)
go func() {
defer pr.Close()
// tar to local disk
tarFile, err := os.OpenFile("/path/to/tar/ball/test.tar.gz", os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
log.Fatal(err)
}
defer tarFile.Close()
if _, err := io.Copy(tarFile, pr); err != nil {
log.Fatal(err)
}
status <- true
}()
err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
header, err := tar.FileInfoHeader(info, info.Name())
if err != nil {
return err
}
// header.Name = strings.TrimPrefix(strings.Replace(path, src, "", -1), string(filepath.Separator))
if err := tarWriter.WriteHeader(header); err != nil {
return err
}
if info.Mode().IsDir() {
return nil
}
fmt.Println(path)
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(tarWriter, f); err != nil {
return err
}
return nil
})
if err != nil {
log.Fatal(err)
}
pw.Close()
<-status
}
You are closing the pipe before the deferred Close calls on the gzipWriter and tarWriter. There's no error, because you're not checking the error on either of those close calls. You need to close the tarWriter, then the gzipWriter, then the PipeWriter, in that order.
However, there's no reason for the pipe at all in this code, and you can remove the goroutine and the associated coordination altogether if you write directly to the file.
tarFile, err := os.OpenFile("/tmp/test.tar.gz", os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
log.Fatal(err)
}
defer tarFile.Close()
gzipWriter := gzip.NewWriter(tarFile)
defer gzipWriter.Close()
tarWriter := tar.NewWriter(gzipWriter)
defer tarWriter.Close()
My Go application outputs some amounts of text data and I need to pipe it to some external command (e.g. less). I haven't find any way to pipe this data to syscall.Exec'ed process.
As a workaround I write that text data to a temporary file and then use that file as an argument to less:
package main
import (
"io/ioutil"
"log"
"os"
"os/exec"
"syscall"
)
func main() {
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
log.Fatal(err)
}
defer os.Remove(tmpfile.Name()) // Never going to happen!
if _, err := tmpfile.Write(content); err != nil {
log.Fatal(err)
}
if err := tmpfile.Close(); err != nil {
log.Fatal(err)
}
binary, err := exec.LookPath("less")
if err != nil {
log.Fatal(err)
}
args := []string{"less", tmpfile.Name()}
if err := syscall.Exec(binary, args, os.Environ()); err != nil {
log.Fatal(err)
}
}
It works but leaves a temporary file on a file system, because syscall.Exec replaces the current Go process with another (less) one and deferred os.Remove won't run. Such behaviour is not desirable.
Is there any way to pipe some data to an external process without leaving any artefacts?
You should be using os/exec to build an exec.Cmd to execute, then you could supply any io.Reader you want as the stdin for the command.
From the example in the documentation:
cmd := exec.Command("tr", "a-z", "A-Z")
cmd.Stdin = strings.NewReader("some input")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
fmt.Printf("in all caps: %q\n", out.String())
If you want to write directly to the command's stdin, then you call cmd.StdInPipe to get an io.WriteCloser you can write to.
If you really need to exec the process in place of your current one, you can simply remove the file before exec'ing, and provide that file descriptor as stdin for the program.
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
log.Fatal(err)
}
os.Remove(tmpfile.Name())
if _, err := tmpfile.Write(content); err != nil {
log.Fatal(err)
}
tmpfile.Seek(0, 0)
err = syscall.Dup2(int(tmpfile.Fd()), syscall.Stdin)
if err != nil {
log.Fatal(err)
}
binary, err := exec.LookPath("less")
if err != nil {
log.Fatal(err)
}
args := []string{"less"}
if err := syscall.Exec(binary, args, os.Environ()); err != nil {
log.Fatal(err)
}
I call imagemagick's convert command with some data I have in memory (from html form upload/web server). This works fine, but I'd like to get the error output of convert in case of an error. How can I do that?
This is my code:
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os/exec"
"path/filepath"
)
func runImagemagick(data []byte, destfilename string) error {
data_buf := bytes.NewBuffer(data)
cmd := exec.Command("convert", "-", destfilename)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = io.Copy(stdin, data_buf)
if err != nil {
return err
}
stdin.Close()
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
func main() {
data, err := ioutil.ReadFile("source.gif")
if err != nil {
log.Fatal(err)
}
err = runImagemagick(data, filepath.Join("/tmp", "abc", "dest.png"))
if err != nil {
log.Fatal(err)
}
}
Now the artificial problem is that the directory /tmp/abc/ does not exist. Normally convert would give me this result:
$ convert - /tmp/abc/foo.png < source.gif
convert: unable to open image `/tmp/abc/foo.png': No such file or directory # error/blob.c/OpenBlob/2617.
convert: WriteBlob Failed `/tmp/abc/foo.png' # error/png.c/MagickPNGErrorHandler/1755.
but I don't "see" this error message within my small program. How can I get the error message and show it to my user?
(And another sub-question is: can you give me an advice if this code looks OK? Are there any obvious flaws in it?)
Pipe stdout and stderr too. For example,
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os/exec"
"path/filepath"
)
func runImagemagick(data []byte, destfilename string) error {
cmd := exec.Command("convert", "-", destfilename)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = io.Copy(stdin, bytes.NewBuffer(data))
if err != nil {
return err
}
stdin.Close()
outData, err := ioutil.ReadAll(stdout)
if err != nil {
return err
}
if len(outData) > 0 {
log.Print(string(outData))
}
errData, err := ioutil.ReadAll(stderr)
if err != nil {
return err
}
if len(errData) > 0 {
log.Print(string(errData))
}
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
func main() {
data, err := ioutil.ReadFile("source.gif")
if err != nil {
log.Fatal(err)
}
err = runImagemagick(data, filepath.Join("/tmp", "abc", "dest.png"))
if err != nil {
log.Fatal(err)
}
}
Output:
2013/03/03 15:02:20 convert.im6: unable to open image `/tmp/abc/dest-0.png': No such file or directory # error/blob.c/OpenBlob/2638.
convert.im6: WriteBlob Failed `/tmp/abc/dest-0.png' # error/png.c/MagickPNGErrorHandler/1728.
2013/03/03 15:02:20 exit status 1
exit status 1
There's no need to use pipes because bytes.Buffer implements the io.Writer interface and so it can be used just fine to collect the program's output:
func runImagemagick(data []byte, destfilename string) error {
cmd := exec.Command("convert", "-", destfilename)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok {
return &imagemagickError{ee, stdout.Bytes(), stderr.Bytes()}
} else {
return err
}
}
if stderr.Len() > 0 {
return errors.New(fmt.Sprintf("imagemagick wrote to stderr: %s", stderr.Bytes()))
}
if stdout.Len() > 0 {
log.Print(stdout.Bytes())
}
return nil
}