Different result when run io.Copy(os.Stdout, &r) consecutively in Golang - go

I am playing around Golang. About io.Copy
I put 2 consecutive io.Copy in the code, but i expect it output twice result(testtesttest). But the 2nd one is nil. Can anyone help explain why? tks
package main
import (
"io"
"os"
"strings"
"fmt"
)
type testReader struct {
w io.Reader
str string
}
func (tt *testReader) Read (b []byte) (n int, err error) {
io.Copy(os.Stdout, tt.w)
n, err = tt.w.Read(b)
if tt.w !=nil {
return 0,io.EOF
}
return
}
func main() {
s := strings.NewReader("testtesttest!!!")
r := testReader{s,"ttthhh"}
fmt.Println(&r)
io.Copy(os.Stdout, &r)
// s.Seek(0,0) // solution from Poy's answer
io.Copy(os.Stdout, &r)
}

I'm going to prune down the given example to (as there is a bit of noise):
package main
import (
"io"
"os"
"strings"
)
func main() {
s := strings.NewReader("testtesttest")
io.Copy(os.Stdout, s) // Will print "testtesttest"
io.Copy(os.Stdout, s) // Won't print anything
}
The reason the second copy won't output anything is the io.Reader (s) has already been read. Reading from a io.Reader is not idempotent (you can't call it twice to get the same results). It also doesn't have a way to "reset" it or anything.
As #JRLambert pointed out you have s.Seek() and s.Reset() to allow you to start reading again.

Quick addition to all the correct answers (#poy and #JRLambert) provided so far... Use io.TeeReader or io.MultiWriter for times when you would want to use io.Copy more than once. Below are some examples of using each.
Using io.TeeReader
package main
import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
)
func main() {
sourceFile, _ := os.Open("source/ebook.pdf")
var buf bytes.Buffer
tee := io.TeeReader(sourceFile, &buf)
process := func(sourceReader io.Reader) {
targetFile, _ := os.Create("target/ebook.pdf")
defer targetFile.Close()
if _, err := io.Copy(targetFile, sourceReader); err != nil {
fmt.Println(err)
}
}
process(tee)
fmt.Println(checksum(&buf))
}
func checksum(buf *bytes.Buffer) string {
h := md5.New()
b, _ := ioutil.ReadAll(buf)
if _, err := h.Write(b); err != nil {
fmt.Println(err)
}
return hex.EncodeToString(h.Sum(nil)[:16])
}
Using io.MultiWriter
package main
import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
)
func main() {
sourceFile, _ := os.Open("source/ebook.pdf")
process := func(sourceReader io.Reader) {
targetFile, _ := os.Create("target/ebook.pdf")
defer targetFile.Close()
var buf1, buf2 bytes.Buffer
w := io.MultiWriter(targetFile, &buf1, &buf2)
if _, err := io.Copy(w, sourceReader); err != nil {
fmt.Println(err)
}
fmt.Println(checksum(&buf1))
fmt.Println(checksum(&buf2))
}
process(sourceFile)
}
func checksum(buf *bytes.Buffer) string {
h := md5.New()
b, _ := ioutil.ReadAll(buf)
if _, err := h.Write(b); err != nil {
fmt.Println(err)
}
return hex.EncodeToString(h.Sum(nil)[:16])
}

Related

Golang segment stdin stream to file

instead of writing a pipe to a huge file i want to segment the stream in chunks on signal USR1. i think i got the basics working but the app just hangs and nothing happens, any clues or best practices when handling with an uncontrollable input stream and byte perfect segmentation?
package main
import (
"bufio"
"fmt"
"io"
"os"
"os/signal"
"syscall"
"time"
)
var done bool
func handle(c chan os.Signal) {
for {
sig := <-c
switch sig {
case syscall.SIGUSR1:
fmt.Println("###Sink temporarily_closed###")
done = true
case syscall.SIGUSR2:
fmt.Println("###Sink closed###")
done = true
case syscall.SIGHUP:
fmt.Println("###Sink running###")
}
}
}
func check(e error) {
if e != nil {
panic(e)
}
}
func main() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
go handle(c)
reader := bufio.NewReaderSize(os.Stdin,1024*10)
for true {
if done {
file, err := os.Create("./temp.file")
check(err)
writer := bufio.NewWriter(file)
written, err := io.Copy(writer,reader)
check(err)
fmt.Println(written)
writer.Flush()
file.Close()
reader.Reset(os.Stdin)
done = false
}
time.Sleep(time.Millisecond)
}
}
So you need to io.CopyN(dst, src, 4096) in the loop and rotate the file once in a while. See example. I made rotation by size but it is easy to add signal handling.
package main
import (
"fmt"
"io"
"log"
"os"
"time"
)
var count int
var f *os.File
func rotate() *os.File {
if f != nil {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
fname := fmt.Sprintf("./dump-%d.bin", count)
count++
f, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
log.Println("rotated:", fname)
return f
}
func main() {
var n, written int
reader := os.Stdin
for {
if written == 0 || written >= 4096*10 {
f = rotate()
written = 0
}
n, err := io.CopyN(f, reader, 4096)
if err != nil {
log.Fatal(err)
}
written += n
log.Println("written:", written)
time.Sleep(time.Millisecond * 500)
}
}

Insert slice into url link as text

I am trying to make a bot which sends mails. I got the following code. I am wondering, if it's possible to place the mails slice into this field seperated by a , ?
&bcc=
If my test.txt contains
test1#mail.com
test2#mail.com
I'd like the part of the link to contain &bcc=test1#mail.com,test2#mail.com
Is this doable with Go?
package main
import (
"bufio"
"fmt"
"log"
"os"
"os/exec"
)
func main() {
file, err := os.Open("test.txt")
if err != nil {
log.Fatal(err)
}
var mails []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
mails = append(mails, scanner.Text())
}
fmt.Println(mails)
exec.Command("xdg-open", "https://mail.google.com/mail/u/0/?fs=1&tf=cm&to=contact#test.com,&bcc=test1#mail.com,test2#mail.com&su=Hello+World!&body=This+Is+Just+An+Example").Run()
}
You can use "fmt.Sprintf()".It solves your problem.
package main
import (
"bufio"
"fmt"
"log"
"os"
)
func main() {
file, err := os.Open("test.txt")
if err != nil {
log.Fatal(err)
}
var mails []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
mails = append(mails, scanner.Text())
}
sendMails := ""
for _, m := range mails {
sendMails += fmt.Sprintf("%s", m)
}
command := fmt.Sprintf("https://mail.google.com/mail/u/0/?fs=1&tf=cm&to=contact#test.com,&bcc=%s&su=Hello+World!&body=This+Is+Just+An+Example", sendMails)
fmt.Println(mails)
exec.Command("xdg-open", command).Run()
}

Writing buffer to file doesn't return error, so why is file empty afterwords?

I'm trying to read all standard input and write it to a file. It's writing nothing to the file provided. Why is it not working?
package main
import (
"os"
"bytes"
"fmt"
"bufio"
)
func main() {
fn := os.Args[1]
var input bytes.Buffer
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
fmt.Fprintf(&input, scanner.Text())
fmt.Fprintf(&input, "\n")
}
fi, _ := os.Open(fn)
defer fi.Close()
fi.Write(input.Bytes())
}
And then...
touch writetothis.txt
echo "input text" | go run main.go writetothis.txt
# writetothis.txt is empty
Open opens a file in read-only mode.
Refer to documentation: https://golang.org/pkg/os/#Open
Instead, use OpenFile.
Also, always check for errors whenever you code. It'll save you at least weeks of work-hours in your lifetime.
Here is a working code:
package main
import (
"bufio"
"bytes"
"fmt"
"os"
)
func main() {
fn := os.Args[1]
var input bytes.Buffer
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
fmt.Fprintf(&input, scanner.Text())
fmt.Fprintf(&input, "\n")
}
fmt.Println(input.Bytes())
fi, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
fmt.Println("Error with Open()",err)
}
defer fi.Close()
n, err := fi.Write(input.Bytes())
if err != nil {
fmt.Println("Error with Write()", err)
}
fmt.Println("Bytes written to file: ",n)
}
In your code you can silently fail because you aren't checking the error. It's likely a path issue. You can change your code slightly and let ioutil handle file creation so that paths aren't so much of an issue. Remember to always check the errs
package main
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
)
func main() {
fn := os.Args[1]
var input bytes.Buffer
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
fmt.Fprintf(&input, scanner.Text())
fmt.Fprintf(&input, "\n")
}
err := ioutil.WriteFile(fn, input.Bytes(), 0644)
if err != nil {
log.Fatal(err)
}
}

Is it possible to do a conversion after assignment?

I'm wondering if there is any trick to avoid the xv identifier/allocation. Basically something like x, err := T1(strconv.Atoi("1"))
package main
import "fmt"
import "strconv"
type T1 int
func main() {
xv, err := strconv.Atoi("1")
if err != nil {
panic(err)
}
x := T1(xv)
fmt.Println(x)
}
For example, only x escapes to heap,
package main
import (
"fmt"
"strconv"
)
type T1 int
func atoi(a string) int {
i, err := strconv.Atoi(a)
if err != nil {
panic(err)
}
return i
}
func main() {
x := T1(atoi("1"))
fmt.Println(x)
}
Output:
1
No, I believe there is no such trick.
When I want to avoid declarations of unnecessary variables in the scope, and for one-off operations, I sometimes use this kind of anonymous function calls:
package main
import "fmt"
import "strconv"
type T1 int
func main() {
x, err := func() (T1, error) {
x, err := strconv.Atoi("1")
return T1(x), err
}()
fmt.Println(err, x)
}
On the other hand, if you need to perform a similar cast on many occasions, and not necessarily always as a result of the same call (like Atoi), you could create a simple function, which would do the conversion and pass through the error:
package main
import "fmt"
import "strconv"
type T1 int
func resToT1(n int, err error) (T1, error) {
return T1(n), err
}
func main() {
x, err := resToT1(strconv.Atoi("1"))
fmt.Println(err, x)
}

Save an image from url to file

Very new to Go (first simple project I'm working on).
Question: How do I get an image from URL and then save it to my computer?
Here's what I have so far:
package main
import (
"fmt"
"net/http"
"image"
"io/ioutil"
)
func main() {
url := "http://i.imgur.com/m1UIjW1.jpg"
// don't worry about errors
response, _ := http.Get(url);
defer response.Body.Close()
m, _, err := image.Decode(response.Body)
error := ioutil.WriteFile("/images/asdf.jpg", m, 0644)
}
However, when I run this code, I get cannot use m (type image.Image) as type []byte in function argument
I'm assuming I have to convert image.Image (variable m) into an undefined amount of bytes? Is that the correct way to go about this?
There is no need to decode the file. Simply copy the response body to a file you've opened. Here's the deal in the modified example:
response.Body is a stream of data, and implements the Reader interface - meaning you can sequentially call Read on it, as if it was an open file.
The file I'm opening here implements the Writer interface. This is the opposite - it's a stream you can call Write on.
io.Copy "patches" a reader and a writer, consumes the reader stream and writes its contents to a Writer.
This is one of my favorite things about go - implicit interfaces. You don't have to declare you're implementing an interface, you just have to implement it to be used in some context. This allows mixing and matching of code that doesn't need to know about other code it's interacting with.
package main
import (
"fmt"
"io"
"log"
"net/http"
"os"
)
func main() {
url := "http://i.imgur.com/m1UIjW1.jpg"
// don't worry about errors
response, e := http.Get(url)
if e != nil {
log.Fatal(e)
}
defer response.Body.Close()
//open a file for writing
file, err := os.Create("/tmp/asdf.jpg")
if err != nil {
log.Fatal(err)
}
defer file.Close()
// Use io.Copy to just dump the response body to the file. This supports huge files
_, err = io.Copy(file, response.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println("Success!")
}
package main
import (
"io"
"net/http"
"os"
"fmt"
)
func main() {
img, _ := os.Create("image.jpg")
defer img.Close()
resp, _ := http.Get("http://i.imgur.com/Dz2r9lk.jpg")
defer resp.Body.Close()
b, _ := io.Copy(img, resp.Body)
fmt.Println("File size: ", b)
}
Try this:
package main
import (
"fmt"
"net/http"
"net/url"
"os"
"strings"
)
var (
fileName string
fullUrlFile string
)
func main() {
fullUrlFile = "https://i.imgur.com/m1UIjW1.jpg"
r, e := http.Get(fullUrlFile)
if e != nil {
panic(e)
}
defer r.Body.Close()
buildFileName()
// Create distination
f, e := os.Create(fileName) // "m1UIjW1.jpg"
if e != nil {
panic(e)
}
defer f.Close()
// Fill distination with content
n, e := f.ReadFrom(r.Body)
if e != nil {
panic(e)
}
fmt.Println("File size: ", n)
}
func buildFileName() {
fileUrl, e := url.Parse(fullUrlFile)
if e != nil {
panic(e)
}
path := fileUrl.Path
segments := strings.Split(path, "/")
fileName = segments[len(segments)-1]
println(fileName)
}
You can also use the ReadFrom method:
package main
import (
"net/http"
"os"
)
func main() {
r, e := http.Get("https://i.imgur.com/m1UIjW1.jpg")
if e != nil {
panic(e)
}
defer r.Body.Close()
f, e := os.Create("m1UIjW1.jpg")
if e != nil {
panic(e)
}
defer f.Close()
f.ReadFrom(r.Body)
}
https://golang.org/pkg/os#File.ReadFrom
What is the type of response.Body? You should just convert that into a []byte if it is not and write that to disk. There is no reason to use the image class unless you have some reason to treat the data as an image. Just treat the data as a series of bytes and write it to the disk.

Resources