Problem
I am sending packet using tcp, with first 8 byte as long which
contains the actual packet length while receiving it, after some point
it gets wrong packet length which causes the following error "slice
out of range" because the packet length that is received is way bigger
but using TCP Dump I can see it is receiving the correct packet size.
Client TCP Code
package main
import (
"fmt"
"net"
"ByteBuffer"
"log"
"sync"
)
func main() {
conn, err := net.Dial("tcp", "192.168.90.116:8300")
if err != nil {
fmt.Println(err)
return
}
byteBuffer := ByteBuffer.Buffer{
Endian:"big",
}
msg := "Hello World"
totalByteLen := len(msg)
byteBuffer.PutLong(totalByteLen)
byteBuffer.Put([]byte(msg))
log.Println(byteBuffer.Array())
for i:=0;i<1000000000000;i++{
go write(conn, byteBuffer.Array())
}
}
var lck = &sync.Mutex{}
func write(conn net.Conn, data []byte){
lck.Lock()
_, err := conn.Write(data)
lck.Unlock()
if err != nil{
return
}
}
Server TCP Code
func HandleRequest(conn net.Conn){
defer conn.Close()
for {
// creating a 8 byte buffer array
sizeBuf := make([]byte, 8)
// reading from tcp sockets
_, err := conn.Read(sizeBuf)
// converting the packet size to int64
packetSize := int64(binary.BigEndian.Uint64(sizeBuf))
log.Println(packetSize)
if packetSize < 0 {
continue
}
// reading more bytes from tcp pipe of packetSize length
/*
Here it catches error as the packet size is incorrect but it throws error after receiving aroung 4-5K messages.
*/
completePacket := make([]byte, packetSize)
_, err = conn.Read(completePacket)
// checking error type
if err == io.EOF{
break
}
if err != nil{
break
}
fmt.Println(completePacket)
}
}
Related
I'm trying to do simple UNIX socket communication with a service using GO. For testing I've created a socket like so:
$ nc -vlU /tmp/sock
Bound on /tmp/sock
Listening on /tmp/sock
And in GO, I net.Dial and try to write something, then read. I see the written data in the nc console, so I know that is working. But the net.Conn.Read operation appears non-blocking and will return immediately with zero length. From everything I've read and the example I've seen, this operation should block.
buf := make([]byte, 0, 4096)
ctl, err := net.Dial("unix", "/tmp/sock")
for {
ctl.Write([]byte("test write\n"))
n, err := ctl.Read(buf)
fmt.Printf("Len:%v, Data:%v, err:%v", n, buf, err)
}
I see the connection come through and data written ...
Connection received on /tmp/sock
test write
test write
test write
test write
...
But the GO console loops without blocking, reporting a zero length and no error
Len:0, Data:[], err:<nil>
Len:0, Data:[], err:<nil>
Len:0, Data:[], err:<nil>
Len:0, Data:[], err:<nil>
...
And of course if I type anything into the nc console nothing interesting happens in the GO program output.
Any idea what I'm doing wrong?
short answer
buf := make([]byte, 0, 4096)
this code make a buffer with len=0!!!
do it like
buf := make([]byte, 4096)
example
this is some example i used, and they all works.
In practice, socket read and write should happen in two goroutines
package main
import (
"bufio"
"fmt"
"io"
"net"
"time"
)
func main() {
buf := make([]byte, 0, 4096)
fmt.Println(buf)
main03()
}
func main01() {
// nc -vlU /tmp/sock0120
ctl, err := net.Dial("unix", "/tmp/sock0120")
if err != nil {
fmt.Println(err)
return
}
reader := bufio.NewReader(ctl)
for {
time.Sleep(time.Second)
ctl.Write([]byte("test write\n"))
msg, err := reader.ReadString('\n')
fmt.Printf("Len:%v, Data:%v, err:%v\n", len(msg), msg, err)
}
}
func main02() {
// nc -vlU /tmp/sock0120
buf := make([]byte, 4)
ctl, err := net.Dial("unix", "/tmp/sock0120")
if err != nil {
fmt.Println(err)
return
}
reader := bufio.NewReader(ctl)
for {
time.Sleep(time.Second)
ctl.Write([]byte("test write\n"))
n, err := io.ReadFull(reader, buf)
fmt.Printf("Len:%v, Data:%v, err:%v\n", n, buf, err)
}
}
func main03() {
// nc -vlU /tmp/sock0120
buf := make([]byte, 4)
ctl, err := net.Dial("unix", "/tmp/sock0120")
if err != nil {
fmt.Println(err)
return
}
for {
time.Sleep(time.Second)
ctl.Write([]byte("test write\n"))
n, err := ctl.Read(buf)
fmt.Printf("Len:%v, Data:%v, err:%v\n", n, buf, err)
}
}
The first TCP connection running on localhost on osx always parses the binary sent to it correctly. Subsequent requests lose the binary data, only seeing the first byte [8]. How have I failed to set up my Reader?
package main
import (
"fmt"
"log"
"net"
"os"
"app/src/internal/handler"
"github.com/golang-collections/collections/stack"
)
func main() {
port := os.Getenv("SERVER_PORT")
s := stack.New()
ln, err := net.Listen("tcp", ":8080")
if err != nil {
log.Fatalf("net.Listen: %v", err)
}
fmt.Println("Serving on " + port)
for {
conn, err := ln.Accept()
// defer conn.Close()
if err != nil {
log.Fatal("ln.Accept")
}
go handler.Handle(conn, s)
}
}
package handler
import (
"fmt"
"io"
"log"
"net"
"github.com/golang-collections/collections/stack"
)
func Handle(c net.Conn, s *stack.Stack) {
fmt.Printf("Serving %s\n", c.RemoteAddr().String())
buf := make([]byte, 0, 256)
tmp := make([]byte, 128)
n, err := c.Read(tmp)
if err != nil {
if err != io.EOF {
log.Fatalf("connection Read() %v", err)
}
return
}
buf = append(buf, tmp[:n]...)
}
log:
Serving [::1]:51699
------------- value ---------------:QCXhoy5t
Buffer Length: 9. First Value: 8
Serving [::1]:51700
------------- value ---------------:
Buffer Length: 1. First Value: 8
Serving [::1]:51701
test sent over:
push random string:
QCXhoy5t
push random string:
GPh0EnbS
push random string:
4kJ0wN0R
The docs for Reader say:
Read reads up to len(p) bytes into p. It returns the number of bytes read (0 <= n
<= len(p)) and any error encountered. Even if Read returns n < len(p), it may use
all of p as scratch space during the call. If some data is available but not
len(p) bytes, Read conventionally returns what is available instead of waiting
for more.
So the most likely cause of your issue is that Read is returning the data available (in this case a single character). You can fix this by using ioutil.ReadAll or performing the read in a loop (the fact the data is being added to a buffer makes it look like that was the original intention) with something like:
for {
n, err := c.Read(tmp)
if err != nil {
if err != io.EOF {
// Note that data might have also been received - you should process that
// if appropriate.
log.Fatalf("connection Read() %v", err)
return
}
break // All data received so process it
}
buf = append(buf, tmp[:n]...)
}
Note: There is no guarantee that any data is received; you should check the length before trying to access it (i.e. buf[0] may panic)
Not able to reset or discard the buffer.
I am trying to get the data over the serial port where I am getting data packet of some fixed length for every 10 seconds. I have an infinite for loop to receive the data packets continuously. After receiving the new data packet I am resetting the buffer but when I receive the next data packet, it overwrites the buffer and I get mixed data packet.
Let say I should receive packet abcdef continuously for every n second. But when I try the following code I am receiving packet bcdefa then after n second cdefab then defabc and so on
package main
import (
"bufio"
"log"
"time"
"github.com/tarm/serial"
)
func main() {
c := &serial.Config{Name: "/dev/ttyUSB0", Baud: 57600}
s, err := serial.OpenPort(c)
if err != nil {
log.Println(err)
return
}
for {
time.Sleep(time.Second / 2)
reader := bufio.NewReader(s)
pck, err := reader.Peek(46)
if err != nil {
log.Println(err)
}
go parse(pck)
reader.Reset(s)
}
}
How do I reset or discard the buffer data effectively so that I will receive the exact data packet.
bare in mind i cant check what i m saying here...
1/ you must not instantiate the bufio reader at each iteration
2/ bufio.Reader.Peek does NOT advance the reader
https://golang.org/pkg/bufio/#Reader.Peek
3/ Unless you get a malformed packet, i think you dont need to reset at all.
4/ Please indent your code at play.golang.org
5/ You are not checking the read error for termination
6/ All package i can found to work with serial ports in go exposes an instance of io.Reader, so it might be useless to use an additional bufio.Reader. I suspect you r using https://godoc.org/github.com/tarm/serial#OpenPort
This is probably not the definitive answer, but it should help.
package main
import (
"io"
"log"
"time"
)
func main() {
s, err := serial.OpenPort(c)
if err != nil {
log.Fatal(err)
}
pck := make([]byte, 46)
for {
time.Sleep(time.Second / 2)
n, err := s.Read(pck)
if err != nil {
if err == io.EOF {
break
}
log.Println(err)
}
pck = pck[:n]
go parse(pck)
}
}
I'm building some server/client application in Go (the language is new to me). I searched a lot and read a whole bunch of different examples but there is still one thing I can't find. Lets say I have a single server client up and running. The client will send some kind of a message to the server and vice versa. Encoding and decoding is done by the package gob.
This example is not my application, it is only a quick example:
package main
import (
"bytes"
"encoding/gob"
"fmt"
"log"
)
type Message struct {
Sender string
Receiver string
Command uint8
Value int64
}
func (message *Message) Set(sender string, receiver string, command uint8, value int64) *Message {
message.Sender = sender
message.Receiver = receiver
message.Command = command
message.Value = value
return message
}
func main() {
var network bytes.Buffer // Stand-in for a network connection
enc := gob.NewEncoder(&network) // Will write to network.
dec := gob.NewDecoder(&network) // Will read from network.
message := new(Message).Set("first", "second", 10, -1)
err := enc.Encode(*message) // send message
if err != nil {
log.Fatal("encode error:", err)
}
var m Message
err = dec.Decode(&m) // receice message
if err != nil {
log.Fatal("decode error:", err)
}
fmt.Printf("%q %q %d %d\n", m.Sender, m.Receiver, m.Command, m.Value)
}
This works fine, but I want the server to block until a new message is received so I can put the receiving process inside a infinite for loop inside a goroutine.
Something like that:
for {
// The server blocks HERE until a message from the client is received
fmt.Println("Received message:")
// Decode the new message
var m Message
err = dec.Decode(&m) // receice message
if err != nil {
log.Fatal("decode error:", err)
}
fmt.Printf("%q %q %d %d\n", m.Sender, m.Receiver, m.Command, m.Value)
}
The gob decoder blocks until it has read a full message or there's an error. The read loop in the question works as is.
working example on the playground
add a length header to the raw tcp stream.
that means, send a 4-bytes-length-header information to server before send the real load. and in server side read 4 bytes, allocate buffer, full read total message, and finally decode.
assume you have a tcp connection conn, in server side we could have:
func getInt(v []byte) int {
var r uint
r = 0
r |= uint(v[0]) << 24
r |= uint(v[1]) << 16
r |= uint(v[2]) << 8
r |= uint(v[3]) << 0
return int(r)
}
buf := make([]byte, 4)
_, err := io.ReadFull(conn, buf)
if err != nil {
return
}
length := getInt(buf)
buf = make([]byte, length)
_, err = io.ReadFull(conn, buf)
if err != nil {
return
}
//do gob decode from `buf` here
you may know client side refer the the server side source I think.
I'm wondering if it's possible to count and print the number of bytes downloaded while the file is being downloaded.
out, err := os.Create("file.txt")
defer out.Close()
if err != nil {
fmt.Println(fmt.Sprint(err))
panic(err)
}
resp, err := http.Get("http://example.com/zip")
defer resp.Body.Close()
if err != nil {
fmt.Println(fmt.Sprint(err))
panic(err)
}
n, er := io.Copy(out, resp.Body)
if er != nil {
fmt.Println(fmt.Sprint(err))
}
fmt.Println(n, "bytes ")
If I understand you correctly, you wish to display the number of bytes read, while the data is transferring. Presumably to maintain some kind of a progress bar or something. In which case, you can use Go's compositional data structures to wrap the reader or writer in a custom io.Reader or io.Writer implementation.
It simply forwards the respective Read or Write call to the underlying stream, while doing some additional work with the (int, error) values returned by them. Here is an example you can run on the Go playground.
package main
import (
"bytes"
"fmt"
"io"
"os"
"strings"
)
// PassThru wraps an existing io.Reader.
//
// It simply forwards the Read() call, while displaying
// the results from individual calls to it.
type PassThru struct {
io.Reader
total int64 // Total # of bytes transferred
}
// Read 'overrides' the underlying io.Reader's Read method.
// This is the one that will be called by io.Copy(). We simply
// use it to keep track of byte counts and then forward the call.
func (pt *PassThru) Read(p []byte) (int, error) {
n, err := pt.Reader.Read(p)
pt.total += int64(n)
if err == nil {
fmt.Println("Read", n, "bytes for a total of", pt.total)
}
return n, err
}
func main() {
var src io.Reader // Source file/url/etc
var dst bytes.Buffer // Destination file/buffer/etc
// Create some random input data.
src = bytes.NewBufferString(strings.Repeat("Some random input data", 1000))
// Wrap it with our custom io.Reader.
src = &PassThru{Reader: src}
count, err := io.Copy(&dst, src)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Println("Transferred", count, "bytes")
}
The output it generates is this:
Read 512 bytes for a total of 512
Read 1024 bytes for a total of 1536
Read 2048 bytes for a total of 3584
Read 4096 bytes for a total of 7680
Read 8192 bytes for a total of 15872
Read 6128 bytes for a total of 22000
Transferred 22000 bytes
The stdlib now provides something like jimt's PassThru: io.TeeReader. It helps simplify things a bit:
// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
Total int64 // Total # of bytes transferred
}
// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
n := len(p)
wc.Total += int64(n)
fmt.Printf("Read %d bytes for a total of %d\n", n, wc.Total)
return n, nil
}
func main() {
// ...
// Wrap it with our custom io.Reader.
src = io.TeeReader(src, &WriteCounter{})
// ...
}
playground
The grab Go package implements progress updates (and many other features) for file downloads.
An example of printing progress updates while a download is in process is included in the following walkthrough: http://cavaliercoder.com/blog/downloading-large-files-in-go.html
You can basically call grab.GetAsync which downloads in a new Go routine and then monitor the BytesTransferred or Progress of the returned grab.Response from the calling thread.
Other answers have explained about PassThru. Just provide a full example with callback function base on Dave Jack's answer.
package main
import (
"fmt"
"io"
"net/http"
"os"
"strconv"
)
// writeCounter counts the number of bytes written to it.
type writeCounter struct {
total int64 // total size
downloaded int64 // downloaded # of bytes transferred
onProgress func(downloaded int64, total int64)
}
// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *writeCounter) Write(p []byte) (n int, e error) {
n = len(p)
wc.downloaded += int64(n)
wc.onProgress(wc.downloaded, wc.total)
return
}
func newWriter(size int64, onProgress func(downloaded, total int64)) io.Writer {
return &writeCounter{total: size, onProgress: onProgress}
}
func main() {
client := http.DefaultClient
url := "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
saveTo := "/Users/tin/Desktop/ForBiggerFun.mp4"
download(client, url, saveTo, func(downloaded, total int64) {
fmt.Printf("Downloaded %d bytes for a total of %d\n", downloaded, total)
})
}
func download(client *http.Client, url, filePath string, onProgress func(downloaded, total int64)) (err error) {
// Create file writer
file, err := os.Create(filePath)
if err != nil {
return
}
defer file.Close()
// Determinate the file size
resp, err := client.Head(url)
if err != nil {
return
}
contentLength := resp.Header.Get("content-length")
length, err := strconv.Atoi(contentLength)
if err != nil {
return
}
// Make request
resp, err = client.Get(url)
if err != nil {
return
}
defer resp.Body.Close()
// pipe stream
body := io.TeeReader(resp.Body, newWriter(int64(length), onProgress))
_, err = io.Copy(file, body)
return err
}
Base #Dave Jack
I add progress value and receiving file data from NC (direct TCP data transfer)
// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
Total int64 // Total # of bytes transferred
Last int64
LastUpdate time.Time
}
// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
n := len(p)
wc.Total += int64(n)
now := time.Now()
duration := now.Sub(wc.LastUpdate).Seconds()
if duration > 1 {
wc.LastUpdate = now
rate := float64(wc.Total-wc.Last) / (duration) / 1024.0
wc.Last = wc.Total
fmt.Printf("Read %d bytes for a total of %d , Rate %.1fKb/s \n", n, wc.Total, rate)
}
return n, nil
}
func Server(dest string) {
outputFile, err := os.Create(dest)
if err != nil {
fmt.Println(err)
}
defer outputFile.Close()
fileWriter := bufio.NewWriter(outputFile)
serverListener, err := net.Listen("tcp", "0.0.0.0:"+PORT)
if err != nil {
fmt.Println(err)
}
defer serverListener.Close()
serverConn, err := serverListener.Accept()
if err != nil {
fmt.Println(err)
}
defer serverConn.Close()
wc := &WriteCounter{}
reader := io.TeeReader(serverConn, wc)
serverConnReader := bufio.NewReaderSize(reader, 32*1024*1024)
io.Copy(fileWriter, serverConnReader)
fileWriter.Flush()
outputFile.Sync()
fmt.Println("Done: Writer")
}