Read stderr after process finished - shell

I call imagemagick's convert command with some data I have in memory (from html form upload/web server). This works fine, but I'd like to get the error output of convert in case of an error. How can I do that?
This is my code:
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os/exec"
"path/filepath"
)
func runImagemagick(data []byte, destfilename string) error {
data_buf := bytes.NewBuffer(data)
cmd := exec.Command("convert", "-", destfilename)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = io.Copy(stdin, data_buf)
if err != nil {
return err
}
stdin.Close()
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
func main() {
data, err := ioutil.ReadFile("source.gif")
if err != nil {
log.Fatal(err)
}
err = runImagemagick(data, filepath.Join("/tmp", "abc", "dest.png"))
if err != nil {
log.Fatal(err)
}
}
Now the artificial problem is that the directory /tmp/abc/ does not exist. Normally convert would give me this result:
$ convert - /tmp/abc/foo.png < source.gif
convert: unable to open image `/tmp/abc/foo.png': No such file or directory # error/blob.c/OpenBlob/2617.
convert: WriteBlob Failed `/tmp/abc/foo.png' # error/png.c/MagickPNGErrorHandler/1755.
but I don't "see" this error message within my small program. How can I get the error message and show it to my user?
(And another sub-question is: can you give me an advice if this code looks OK? Are there any obvious flaws in it?)

Pipe stdout and stderr too. For example,
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os/exec"
"path/filepath"
)
func runImagemagick(data []byte, destfilename string) error {
cmd := exec.Command("convert", "-", destfilename)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = io.Copy(stdin, bytes.NewBuffer(data))
if err != nil {
return err
}
stdin.Close()
outData, err := ioutil.ReadAll(stdout)
if err != nil {
return err
}
if len(outData) > 0 {
log.Print(string(outData))
}
errData, err := ioutil.ReadAll(stderr)
if err != nil {
return err
}
if len(errData) > 0 {
log.Print(string(errData))
}
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
func main() {
data, err := ioutil.ReadFile("source.gif")
if err != nil {
log.Fatal(err)
}
err = runImagemagick(data, filepath.Join("/tmp", "abc", "dest.png"))
if err != nil {
log.Fatal(err)
}
}
Output:
2013/03/03 15:02:20 convert.im6: unable to open image `/tmp/abc/dest-0.png': No such file or directory # error/blob.c/OpenBlob/2638.
convert.im6: WriteBlob Failed `/tmp/abc/dest-0.png' # error/png.c/MagickPNGErrorHandler/1728.
2013/03/03 15:02:20 exit status 1
exit status 1

There's no need to use pipes because bytes.Buffer implements the io.Writer interface and so it can be used just fine to collect the program's output:
func runImagemagick(data []byte, destfilename string) error {
cmd := exec.Command("convert", "-", destfilename)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok {
return &imagemagickError{ee, stdout.Bytes(), stderr.Bytes()}
} else {
return err
}
}
if stderr.Len() > 0 {
return errors.New(fmt.Sprintf("imagemagick wrote to stderr: %s", stderr.Bytes()))
}
if stdout.Len() > 0 {
log.Print(stdout.Bytes())
}
return nil
}

Related

How to work with tar command using golang exec.Cmd

I have a tar.gz file and i need to unpack it using golang.
I've tried libs like "archive/tar" but they gave me error: archive/tar: invalid tar header. Now my idea was to use exec to run tar command and unpack tarball, but it always exits with code 2.
My code:
func unpack(tarName string) error {
path, _ := os.Getwd()
//err := Untar(path+"/"+tarName, path+"/")
fmt.Printf(path + "/" + tarName)
cmd := exec.Command("tar", "-xfv", path+"/"+tarName)
cmd.Stdout = os.Stdout
err := cmd.Run()
return err
}
If you are trying to compress tar.gz file you need first to decompress gzip.
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"log"
"os"
)
func ExtractTarGz(gzipStream io.Reader) {
uncompressedStream, err := gzip.NewReader(gzipStream)
if err != nil {
log.Fatal("ExtractTarGz: NewReader failed")
}
tarReader := tar.NewReader(uncompressedStream)
for true {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("ExtractTarGz: Next() failed: %s", err.Error())
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.Mkdir(header.Name, 0755); err != nil {
log.Fatalf("ExtractTarGz: Mkdir() failed: %s", err.Error())
}
case tar.TypeReg:
outFile, err := os.Create(header.Name)
if err != nil {
log.Fatalf("ExtractTarGz: Create() failed: %s", err.Error())
}
if _, err := io.Copy(outFile, tarReader); err != nil {
log.Fatalf("ExtractTarGz: Copy() failed: %s", err.Error())
}
outFile.Close()
default:
log.Fatalf(
"ExtractTarGz: uknown type: %s in %s",
header.Typeflag,
header.Name)
}
}
}
func main() {
r, err := os.Open("./file.tar.gz")
if err != nil {
fmt.Println("error")
}
ExtractTarGz(r)
}

Why is the file empty after writing to it with bufio.Writer?

file, err := os.OpenFile("filename.db", os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatal(err)
}
defer file.Close()
res := 0
writer := bufio.NewWriter(file)
for _, data := range manager {
bin, err := json.Marshal(data)
if err != nil {
log.Println(err)
return
}
res++
if debug {
log.Println(res)
}
fmt.Printf("%s\n", bin)
_, err = writer.Write(bin)
if err != nil {
log.Println(err)
}
_, _ = writer.WriteRune('\n')
}
playground
full code
The file filename.db is created (if didn't exist), but ...is empty...
Why could this happen?
Why is the file empty?
I tried this both on my home pc and a linux server
And in both cases it's empty
As per the suggestion from comment using writer.Flush results in foo and bar values being written in to the document filename.db.
package main
import (
"bufio"
"encoding/json"
"fmt"
"log"
"os"
)
type Valuable struct {
Value string `json:"value"`
}
var debug = true
var manager []Valuable
func main() {
manager = append(manager, Valuable{"foo"}, Valuable{"bar"})
file, err := os.OpenFile("filename.db", os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatal(err)
}
defer file.Close()
res := 0
writer := bufio.NewWriter(file)
defer writer.Flush()
for _, data := range manager {
bin, err := json.Marshal(data)
if err != nil {
log.Println(err)
return
}
res++
if debug {
log.Println(res)
}
fmt.Printf("%s\n", bin)
_, err = writer.Write(bin)
if err != nil {
log.Println(err)
}
_, _ = writer.WriteRune('\n')
}
}

Having issues with multipart.NewWriter using io.pipe

I'm having issues with requests being sent randomly empty. It doesn't always happen, but sometimes out of the blue, it will not send any of the multipart fields. I thought it might have to do with the upload server, so I created a local upload server to print out the request that's being sent, and it comes out empty.
I have added error checks everywhere, but no errors are being returned.
I tried to run the code with -race, but no race condition has been reported.
Edit: Update the code to use CloseWithError()
package main
import (
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"os"
)
var (
upload_url string = "https://upload.imagekit.io/api/v1/files/upload"
file_name string = "favicon-516140983.ico"
api_secret_key string = "PRIVATE_KEY"
)
func UploadMultipartFile(client *http.Client, uri, key, path string) (*http.Response, error) {
body, writer := io.Pipe()
req, err := http.NewRequest(http.MethodPost, uri, body)
if err != nil {
log.Println(err)
return nil, err
}
mwriter := multipart.NewWriter(writer)
req.Header.Add("Content-Type", mwriter.FormDataContentType())
req.SetBasicAuth(api_secret_key, "")
go func() {
var err error
defer func() {
if err != nil {
writer.CloseWithError(err)
} else {
writer.Close()
}
}()
var file *os.File
file, err = os.Open(path)
if err != nil {
return
}
defer file.Close()
if err = mwriter.WriteField("fileName", file_name); err != nil {
return
}
var w io.Writer
w, err = mwriter.CreateFormFile("file", path)
if err != nil {
return
}
var written int64
if written, err = io.Copy(w, file); err != nil {
err = fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err)
return
}
if err = mwriter.Close(); err != nil {
return
}
}()
resp, err := client.Do(req)
if err != nil {
return nil, err
}
return resp, nil
}
func main() {
path, _ := os.Getwd()
path += "/" + file_name
client := &http.Client{}
resp, err := UploadMultipartFile(client, upload_url, "file", path)
if err != nil {
log.Println(err)
} else {
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
_, err := io.Copy(os.Stdout, resp.Body)
if err != nil {
log.Fatal(err)
}
resp.Body.Close()
}
}

golang os.Close() function works, but os.Remove() function does not

I am trying to create a file, open it, do some processing on it & close it. Finally, I want to delete the file.
All these operations are executed successfully, except the deletion.
My code is:
package main
import (
"fmt"
"os"
"log"
)
func main() {
fmt.Println("Hello")
metaFileName := "./metadata.txt"
_, err2 := os.Create(metaFileName)
if err2 != nil {
log.Fatal(err2)
}
openMetaFile, err := os.Open(metaFileName)
if err != nil {
log.Fatal(err)
}
err = openMetaFile.Close()
if err != nil {
log.Fatal(err)
}
err = os.Remove(metaFileName)
if err != nil {
log.Fatal(err)
}
fmt.Println("Success")
}
The output is:
Hello
2020/08/24 00:00:00 remove ./metadata.txt: The process cannot access the file be
cause it is being used by another process.
I am clueless about this
The problem is related to the first opened file.
package main
import (
"fmt"
"log"
"os"
)
const metaFileName = "./metadata.txt"
func main() {
var (
err error
tmpFile, openMetaFile *os.File
)
fmt.Println("Hello")
if tmpFile, err = os.Create(metaFileName); err != nil {
log.Fatal(err)
}
if err = tmpFile.Close(); err != nil {
log.Fatal(err)
}
if openMetaFile, err = os.Open(metaFileName); err != nil {
log.Fatal(err)
}
if err = openMetaFile.Close(); err != nil {
log.Fatal(err)
}
if err = os.Remove(metaFileName); err != nil {
log.Fatal(err)
}
fmt.Println("Success")
}
As you can see, i've used the var a in order to close the first file that you have opened. The result is following one:
Hello
Success

Saving a continuous stream of images from ffmpeg image2pipe

I am trying to save a sequence/continuous images from ffmpeg image2pipe in go. The problem with the code below that it does only save the first image in the stream due to the blocking nature of io.Copy since it waits for the reader or the writer to close.
package main
import (
"fmt"
"io"
"log"
"os"
"os/exec"
"strconv"
"time"
)
//Trying to get png from stdout pipe
func main() {
fmt.Println("Running the camera stream")
ffmpegCmd := exec.Command("ffmpeg", "-loglevel", "quiet", "-y", "-rtsp_transport", "tcp", "-i", "rtsp://admin:123456#192.168.1.41:554/h264Preview_01_main", "-r", "1", "-f", "image2pipe", "pipe:1")
ffmpegOut, err := ffmpegCmd.StdoutPipe()
if err != nil {
return
}
err = ffmpegCmd.Start()
if err != nil {
log.Fatal(err)
}
count := 0
for {
count++
t := time.Now()
fmt.Println("writing image" + strconv.Itoa(count))
filepath := "image-" + strconv.Itoa(count) + "-" + t.Format("20060102150405.png")
out, err := os.Create(filepath)
if err != nil {
log.Fatal(err)
}
defer out.Close()
_, err = io.Copy(out, ffmpegOut)
if err != nil {
log.Fatalf("unable to copy to file: %s", err.Error())
}
}
if err := ffmpegCmd.Wait(); err != nil {
log.Fatal("Error while waiting:", err)
}
}
I implemented my own save and copy function based on the io.Copy code https://golang.org/src/io/io.go
func copyAndSave(w io.Writer, r io.Reader) error {
buf := make([]byte, 1024, 1024)
for {
n, err := r.Read(buf[:])
if n == 0 {
}
if n > 0 {
d := buf[:n]
_, err := w.Write(d)
if err != nil {
return err
}
}
if err != nil {
return err
}
}
return nil
}
then I updated the for loop in my main function to the below block but still I am only getting the first image in the sequence. due to r.Read(buf[:]) is being a blocking call.
for {
count++
t := time.Now()
fmt.Println("writing image" + strconv.Itoa(count))
filepath := "image-" + strconv.Itoa(count) + "-" + t.Format("20060102150405.png")
out, err := os.Create(filepath)
if err != nil {
log.Fatal(err)
}
defer out.Close()
err = copyAndSave(out, ffmpegOut)
if err != nil {
if err == io.EOF {
break
}
log.Fatalf("unable to copy to file: %s", err.Error())
break
}
}

Resources