Consider the following benchmark:
package main
import (
"io/ioutil"
"os"
"os/exec"
"testing"
)
func BenchmarkRun(b *testing.B) {
for i := 0; i < b.N; i++ {
source := `package main
import "fmt"
func main() {
fmt.Println("foo")
}`
if err := ioutil.WriteFile("cmd.go", []byte(source), 0777); err != nil {
b.Error(err)
}
defer os.Remove("cmd.go")
if err := exec.Command("go", "run", "cmd.go").Run(); err != nil {
b.Error(err)
}
}
}
This takes around 0.3sec per operation.
Is there any way of speeding up a compile / run cycle?
It seems clumsy to write a temporary file and exec go run. Is there a way to invoke the compiler without doing this?
You can always create a binary and use it later on.
Example:
package main
import (
"io/ioutil"
"os"
"os/exec"
"path"
"testing"
)
func BenchmarkRun(b *testing.B) {
tmpdir, err := ioutil.TempDir("", "go-bench-")
if err != nil {
b.Fatal(err)
}
defer os.RemoveAll(tmpdir)
source := `package main
import "fmt"
func main() {
fmt.Println("foo")
}`
if err := ioutil.WriteFile(path.Join(tmpdir, "cmd.go"), []byte(source), 0777); err != nil {
b.Fatal(err)
}
defer os.Remove(path.Join(tmpdir, "cmd.go"))
cmd := exec.Command("go", "build", "-o", "cmd", ".")
cmd.Dir = tmpdir
if err := cmd.Run(); err != nil {
b.Fatal(err)
}
defer os.Remove(path.Join(tmpdir, "cmd"))
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := exec.Command(path.Join(tmpdir, "cmd")).Run(); err != nil {
b.Error(err)
}
}
}
Related
There is a program which creates file per second. I want to append file names into slice and print them. Now my program executes incorrect, it appends names but only for one file name. So I expect to get []string{"1","2","3"}, instead I get []string{"1","1","1"}, []string{"2","2","2"}, []string{"3","3","3"}. How to correct my prog to get expected result?
package main
import (
"encoding/csv"
"fmt"
"os"
"strconv"
"time"
)
func main() {
for {
time.Sleep(1 * time.Second)
createFile()
}
}
func createFile() {
rowFile := time.Now().Second()
fileName := strconv.Itoa(rowFile)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
fmt.Println(err)
}
defer file.Close()
writer := csv.NewWriter(file)
writer.Comma = '|'
err = writer.Write([]string{""})
if err != nil {
fmt.Println(err)
}
countFiles(fileName)
}
func countFiles(fileName string) {
arrFiles := make([]string, 0, 3)
for i := 0; i < 3; i++ {
arrFiles = append(arrFiles, fileName)
}
fmt.Println(arrFiles)// here I expect ["1","2","3"] then ["4","5","6"] and so on. But now there is ["1","1","1"] then ["2","2","2"] and so on
}
createFile() does not persist created file names in any way. You can do something like that:
package main
import (
"encoding/csv"
"fmt"
"os"
"strconv"
"time"
)
func main() {
files := []string{}
for {
time.Sleep(1 * time.Second)
files = append(files, createFile())
fmt.Println(files)
}
}
func createFile() string {
rowFile := time.Now().Second()
fileName := strconv.Itoa(rowFile)
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
fmt.Println(err)
}
defer file.Close()
writer := csv.NewWriter(file)
writer.Comma = '|'
err = writer.Write([]string{""})
if err != nil {
fmt.Println(err)
}
return fileName
}
instead of writing a pipe to a huge file i want to segment the stream in chunks on signal USR1. i think i got the basics working but the app just hangs and nothing happens, any clues or best practices when handling with an uncontrollable input stream and byte perfect segmentation?
package main
import (
"bufio"
"fmt"
"io"
"os"
"os/signal"
"syscall"
"time"
)
var done bool
func handle(c chan os.Signal) {
for {
sig := <-c
switch sig {
case syscall.SIGUSR1:
fmt.Println("###Sink temporarily_closed###")
done = true
case syscall.SIGUSR2:
fmt.Println("###Sink closed###")
done = true
case syscall.SIGHUP:
fmt.Println("###Sink running###")
}
}
}
func check(e error) {
if e != nil {
panic(e)
}
}
func main() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
go handle(c)
reader := bufio.NewReaderSize(os.Stdin,1024*10)
for true {
if done {
file, err := os.Create("./temp.file")
check(err)
writer := bufio.NewWriter(file)
written, err := io.Copy(writer,reader)
check(err)
fmt.Println(written)
writer.Flush()
file.Close()
reader.Reset(os.Stdin)
done = false
}
time.Sleep(time.Millisecond)
}
}
So you need to io.CopyN(dst, src, 4096) in the loop and rotate the file once in a while. See example. I made rotation by size but it is easy to add signal handling.
package main
import (
"fmt"
"io"
"log"
"os"
"time"
)
var count int
var f *os.File
func rotate() *os.File {
if f != nil {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
fname := fmt.Sprintf("./dump-%d.bin", count)
count++
f, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
log.Println("rotated:", fname)
return f
}
func main() {
var n, written int
reader := os.Stdin
for {
if written == 0 || written >= 4096*10 {
f = rotate()
written = 0
}
n, err := io.CopyN(f, reader, 4096)
if err != nil {
log.Fatal(err)
}
written += n
log.Println("written:", written)
time.Sleep(time.Millisecond * 500)
}
}
I am playing around Golang. About io.Copy
I put 2 consecutive io.Copy in the code, but i expect it output twice result(testtesttest). But the 2nd one is nil. Can anyone help explain why? tks
package main
import (
"io"
"os"
"strings"
"fmt"
)
type testReader struct {
w io.Reader
str string
}
func (tt *testReader) Read (b []byte) (n int, err error) {
io.Copy(os.Stdout, tt.w)
n, err = tt.w.Read(b)
if tt.w !=nil {
return 0,io.EOF
}
return
}
func main() {
s := strings.NewReader("testtesttest!!!")
r := testReader{s,"ttthhh"}
fmt.Println(&r)
io.Copy(os.Stdout, &r)
// s.Seek(0,0) // solution from Poy's answer
io.Copy(os.Stdout, &r)
}
I'm going to prune down the given example to (as there is a bit of noise):
package main
import (
"io"
"os"
"strings"
)
func main() {
s := strings.NewReader("testtesttest")
io.Copy(os.Stdout, s) // Will print "testtesttest"
io.Copy(os.Stdout, s) // Won't print anything
}
The reason the second copy won't output anything is the io.Reader (s) has already been read. Reading from a io.Reader is not idempotent (you can't call it twice to get the same results). It also doesn't have a way to "reset" it or anything.
As #JRLambert pointed out you have s.Seek() and s.Reset() to allow you to start reading again.
Quick addition to all the correct answers (#poy and #JRLambert) provided so far... Use io.TeeReader or io.MultiWriter for times when you would want to use io.Copy more than once. Below are some examples of using each.
Using io.TeeReader
package main
import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
)
func main() {
sourceFile, _ := os.Open("source/ebook.pdf")
var buf bytes.Buffer
tee := io.TeeReader(sourceFile, &buf)
process := func(sourceReader io.Reader) {
targetFile, _ := os.Create("target/ebook.pdf")
defer targetFile.Close()
if _, err := io.Copy(targetFile, sourceReader); err != nil {
fmt.Println(err)
}
}
process(tee)
fmt.Println(checksum(&buf))
}
func checksum(buf *bytes.Buffer) string {
h := md5.New()
b, _ := ioutil.ReadAll(buf)
if _, err := h.Write(b); err != nil {
fmt.Println(err)
}
return hex.EncodeToString(h.Sum(nil)[:16])
}
Using io.MultiWriter
package main
import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
)
func main() {
sourceFile, _ := os.Open("source/ebook.pdf")
process := func(sourceReader io.Reader) {
targetFile, _ := os.Create("target/ebook.pdf")
defer targetFile.Close()
var buf1, buf2 bytes.Buffer
w := io.MultiWriter(targetFile, &buf1, &buf2)
if _, err := io.Copy(w, sourceReader); err != nil {
fmt.Println(err)
}
fmt.Println(checksum(&buf1))
fmt.Println(checksum(&buf2))
}
process(sourceFile)
}
func checksum(buf *bytes.Buffer) string {
h := md5.New()
b, _ := ioutil.ReadAll(buf)
if _, err := h.Write(b); err != nil {
fmt.Println(err)
}
return hex.EncodeToString(h.Sum(nil)[:16])
}
Is there an easy way to get the permanent MAC Address using Go?
package main
import (
"fmt"
"log"
"net"
)
func getMacAddr() ([]string, error) {
ifas, err := net.Interfaces()
if err != nil {
return nil, err
}
var as []string
for _, ifa := range ifas {
a := ifa.HardwareAddr.String()
if a != "" {
as = append(as, a)
}
}
return as, nil
}
func main() {
as, err := getMacAddr()
if err != nil {
log.Fatal(err)
}
for _, a := range as {
fmt.Println(a)
}
}
How can I pipe several external commands together in Go? I've tried this code but I get an error that says exit status 1.
package main
import (
"io"
"log"
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
stdout1, err := c1.StdoutPipe()
if err != nil {
log.Fatal(err)
}
if err = c1.Start(); err != nil {
log.Fatal(err)
}
if err = c1.Wait(); err != nil {
log.Fatal(err)
}
c2 := exec.Command("wc", "-l")
c2.Stdin = stdout1
stdout2, err := c2.StdoutPipe()
if err != nil {
log.Fatal(err)
}
if err = c2.Start(); err != nil {
log.Fatal(err)
}
if err = c2.Wait(); err != nil {
log.Fatal(err)
}
io.Copy(os.Stdout, stdout2)
}
For simple scenarios, you could use this approach:
bash -c "echo 'your command goes here'"
For example, this function retrieves the CPU model name using piped commands:
func getCPUmodel() string {
cmd := "cat /proc/cpuinfo | egrep '^model name' | uniq | awk '{print substr($0, index($0,$4))}'"
out, err := exec.Command("bash","-c",cmd).Output()
if err != nil {
return fmt.Sprintf("Failed to execute command: %s", cmd)
}
return string(out)
}
StdoutPipe returns a pipe that will be connected to the command's
standard output when the command starts. The pipe will be closed
automatically after Wait sees the command exit.
(from http://golang.org/pkg/os/exec/#Cmd.StdinPipe )
The fact you do c1.Wait closes the stdoutPipe.
I made a working example (just a demo, add error catching!) :
package main
import (
"bytes"
"io"
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
c2 := exec.Command("wc", "-l")
r, w := io.Pipe()
c1.Stdout = w
c2.Stdin = r
var b2 bytes.Buffer
c2.Stdout = &b2
c1.Start()
c2.Start()
c1.Wait()
w.Close()
c2.Wait()
io.Copy(os.Stdout, &b2)
}
package main
import (
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
c2 := exec.Command("wc", "-l")
c2.Stdin, _ = c1.StdoutPipe()
c2.Stdout = os.Stdout
_ = c2.Start()
_ = c1.Run()
_ = c2.Wait()
}
Like the first answer but with the first command started and waited for in a goroutine. This keeps the pipe happy.
package main
import (
"io"
"os"
"os/exec"
)
func main() {
c1 := exec.Command("ls")
c2 := exec.Command("wc", "-l")
pr, pw := io.Pipe()
c1.Stdout = pw
c2.Stdin = pr
c2.Stdout = os.Stdout
c1.Start()
c2.Start()
go func() {
defer pw.Close()
c1.Wait()
}()
c2.Wait()
}
This is a fully working example. The Execute function takes any number of exec.Cmd instances (using a variadic function) and then loops over them correctly attaching the output of stdout to the stdin of the next command. This must be done before any function is called.
The call function then goes about calling the commands in a loop, using defers to call recursively and ensuring proper closure of pipes
package main
import (
"bytes"
"io"
"log"
"os"
"os/exec"
)
func Execute(output_buffer *bytes.Buffer, stack ...*exec.Cmd) (err error) {
var error_buffer bytes.Buffer
pipe_stack := make([]*io.PipeWriter, len(stack)-1)
i := 0
for ; i < len(stack)-1; i++ {
stdin_pipe, stdout_pipe := io.Pipe()
stack[i].Stdout = stdout_pipe
stack[i].Stderr = &error_buffer
stack[i+1].Stdin = stdin_pipe
pipe_stack[i] = stdout_pipe
}
stack[i].Stdout = output_buffer
stack[i].Stderr = &error_buffer
if err := call(stack, pipe_stack); err != nil {
log.Fatalln(string(error_buffer.Bytes()), err)
}
return err
}
func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) {
if stack[0].Process == nil {
if err = stack[0].Start(); err != nil {
return err
}
}
if len(stack) > 1 {
if err = stack[1].Start(); err != nil {
return err
}
defer func() {
if err == nil {
pipes[0].Close()
err = call(stack[1:], pipes[1:])
}
}()
}
return stack[0].Wait()
}
func main() {
var b bytes.Buffer
if err := Execute(&b,
exec.Command("ls", "/Users/tyndyll/Downloads"),
exec.Command("grep", "as"),
exec.Command("sort", "-r"),
); err != nil {
log.Fatalln(err)
}
io.Copy(os.Stdout, &b)
}
Available in this gist
https://gist.github.com/tyndyll/89fbb2c2273f83a074dc
A good point to know is that shell variables like ~ are not interpolated
I wanted to pipe some video and audio to FFplay. This worked for me:
package main
import (
"io"
"os/exec"
)
func main() {
ffmpeg := exec.Command(
"ffmpeg", "-i", "247.webm", "-i", "251.webm", "-c", "copy", "-f", "webm", "-",
)
ffplay := exec.Command("ffplay", "-")
ffplay.Stdin, ffmpeg.Stdout = io.Pipe()
ffmpeg.Start()
ffplay.Run()
}
https://golang.org/pkg/io#Pipe
package main
import (
...
pipe "github.com/b4b4r07/go-pipe"
)
func main() {
var b bytes.Buffer
pipe.Command(&b,
exec.Command("ls", "/Users/b4b4r07/Downloads"),
exec.Command("grep", "Vim"),
)
io.Copy(os.Stdout, &b)
}
I spent a good day trying to use Denys Séguret answer to come up with a wrapper for multiple exec.Command before I came across this neat package by b4b4r07.
Because it can be complex to build such command chains I have decided to implements a litte go library for that purpose: https://github.com/rainu/go-command-chain
package main
import (
"bytes"
"fmt"
"github.com/rainu/go-command-chain"
)
func main() {
output := &bytes.Buffer{}
err := cmdchain.Builder().
Join("ls").
Join("wc", "-l").
Finalize().WithOutput(output).Run()
if err != nil {
panic(err)
}
fmt.Printf("Errors found: %s", output)
}
With the help of this lib you can also configure std-error forwarding and other things.