How to write and read concurrently on single TCP connection in golang? - go

I try to create a sigle TCP connection client used for multiple write and read concurrently. For example the TCP Server will return value like you write.
The problem is the data exchanged between write and read. I have tried with sync.Mutex but it still doesn't work.
My Result is :
2018/03/10 12:52:10 STRING 4:1
2018/03/10 12:52:10 STRING 5:6
2018/03/10 12:52:10 STRING 2:3
and so on
My Expectation is :
2018/03/10 12:52:10 STRING 4:4
2018/03/10 12:52:10 STRING 5:5
2018/03/10 12:52:10 STRING 2:2
2018/03/10 12:52:10 STRING 3:3
Here's my code :
package main
import (
"bufio"
"log"
"net"
"strconv"
"sync"
"time"
)
type Cc struct {
mux sync.Mutex
rw *bufio.ReadWriter
}
func main() {
addr := "127.0.0.1:3333"
log.Println("Dial " + addr)
conn, err := net.Dial("tcp", addr)
cc := Cc{
rw: bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)),
}
if err != nil {
log.Println(err)
}
for i := 1; i <= 10; i++ {
go testC(cc, strconv.Itoa(i))
}
time.Sleep(60 * time.Second)
}
func testC(cc Cc, st string) {
cc.mux.Lock()
defer cc.mux.Unlock()
_, err := cc.rw.WriteString(st + "\n")
if err != nil {
log.Println(err)
// return errors.Wrap(err, "Could not send additional STRING data ("+strconv.Itoa(n)+" bytes written)")
}
// log.Println("Flush the buffer."s)
err = cc.rw.Flush()
if err != nil {
log.Println(err)
// return errors.Wrap(err, "Flush failed.")
}
time.Sleep(3 * time.Second)
// Read the reply.
// log.Println("Read the reply.")
response, err := cc.rw.ReadString('\n')
if err != nil {
log.Println(err)
// return errors.Wrap(err, "Client: Failed to read the reply: '"+response+"'")
}
log.Println("STRING " + st + ":" + response)
// cc.mux.Unlock()
}
Sorry for my bad English, Thank You.

func testC(cc Cc, st string) actually takes a copy of cc. So, even with locking on a mutex, you actually locking on 10 copies of it which are totally independent. So, your locking code has no effect.
Try changing testC function signature to func testC(cc *Cc, st string). And then your call go testC(&cc, strconv.Itoa(i)).

Related

Failure in creation of simple TCP server that accepts variable message lengths

I am trying to create a tcp server/client connection in golang. I am sending a message length so the server will accept multiple messages until the message length is reached. When I try to read the message header to determine the message length, there is no output. The code seems to gets stuck in the server's tcpreader function's for loop when "conn.read(b)" is called. What might be causing this failure?
My server code:
package main
import (
"bufio"
"bytes"
"encoding/binary"
"fmt"
"log"
"net"
"strings"
)
type hdr struct {
magicNum uint64
msgLen uint32
msgType uint16
padding uint16
}
func tcpReader(conn net.Conn) {
foundLength := false
var hdr struct {
magicNum uint64
msgLen uint32
msgType uint16
padding uint16
}
hdrSize := 16
fmt.Println("tcpReader() header size %v", hdrSize)
//localBuffer := new(bytes.Buffer)
defer conn.Close()
for {
if !foundLength {
fmt.Println("tcpReader() Getting the lenght of the message")
var b = make([]byte, hdrSize)
// where the code seems to get stuck
read, err := conn.Read(b)
if err != nil {
fmt.Println("Failed to read header size %v", hdrSize)
fmt.Println(err)
continue
}
fmt.Println("tcpReader() read header bytes %v %v x", read, string(b))
if read != hdrSize { //add logic check magic number etc
fmt.Println("invalid header")
continue
}
err = binary.Read(bytes.NewReader(b[:16]), binary.BigEndian, &hdr)
if err != nil {
log.Fatal(err)
}
fmt.Println("tcpReader() read header bytes %v", hdr.msgLen)
foundLength = true
// messageLength = int(binary.BigEndian.Uint64(b))
} else {
var message = make([]byte, hdr.msgLen)
read, err := conn.Read(message)
if err != nil {
fmt.Println(err)
continue
}
fmt.Println("Received bytes: %v", read)
if read != int(hdr.msgLen) {
//Add logic to append the read message and update the readSofarBytes etc
fmt.Println("invalid data")
continue
}
fmt.Println("Received:", string(message))
foundLength = false
}
}
}
func main() {
fmt.Println("Launching server...")
fmt.Println("Listen on port")
/* addr, err := net.ResolveTCPAddr("ip4", "127.0.0.1:8081")
if err != nil {
panic(err)
}*/
ln, err := net.Listen("tcp", "127.0.0.1:8081")
if err != nil {
log.Fatal(err)
}
defer ln.Close()
fmt.Println("Accept connection on port")
for {
conn, err := ln.Accept()
if err != nil {
log.Fatal(err)
}
fmt.Println("Calling handleConnection")
//go handleConnection(conn)
go tcpReader(conn)
}
}
My client code:
package main
import (
"bufio"
"bytes"
"encoding/binary"
"fmt"
"net"
)
type hdr struct {
magicNum uint64
msgLen int
msgType uint16
padding uint16
msg string
}
func main() {
var bin_buf bytes.Buffer
msg := hdr{magicNum: 123456789,
msgLen: 60,
msgType: 2,
padding: 0,
msg: "Hello world this is tcp message with big header"}
msg.msgLen = int(len(msg.msg))
fmt.Printf("message length %v\n", msg.msgLen)
binary.Write(&bin_buf, binary.BigEndian, msg)
addr, _ := net.ResolveTCPAddr("tcp", ":8081")
conn, err := net.DialTCP("tcp", nil, addr)
if err != nil {
panic(err.Error())
}
// conn.Write(bin_buf.Bytes())
conn.Write(bin_buf.Bytes())
//conn.Write([]byte("Hello from Clinet\n"))
message, _ := bufio.NewReader(conn).ReadString('\n')
fmt.Print(message)
conn.Close()
}
It is because you are not writing anything from the client side at all. You should never ignore error handling in golang.Change these lines and you'll see where the error is:
err := binary.Write(&bin_buf, binary.BigEndian, msg)
if err != nil {
panic(err.Error())
}
Client is writing 0 bytes to connection because there aren't any
panic: binary.Write: invalid type main.hdr
To complete Keser's answer, you can't use a struct whose fields have types without a fixed size. In this case your hdr struct has two fields that can't be written using binary.Write, the message field with type string and msgLen without specifying the int size. You will have to manually write the string message as it is up to the developer to figure out how to handle strings when writing it through a TCP connection. Additionally, all the fields of your struct are unexported, binary.Write will not be able to access them. Use an uppercase letter to export them.
If you try it, you'll know it.
func main() {
...
err := binary.Write(&bin_buf, binary.BigEndian, msg)
fmt.Println(err)
fmt.Println(string(bin_buf.Bytes()))
...
}

Go udp connection dial once, and send many

please consider this code below, it is a simplified version of a service. I launch a no. of goroutines as needed during its lifetime, and as they go about doing things, they need to send udp messages to a set destination.
package main
import (
"fmt"
"log"
"net"
"time"
)
const (
udp_dest = "192.168.1.200:514"
)
func main() {
fmt.Println("Hello")
message := "this is a test"
log_message(&message)
go worker(1)
go worker(2)
go worker(3)
go worker(4)
time.Sleep(3009 * time.Second)
}
func log_message(message *string) {
RemoteAddr, err := net.ResolveUDPAddr("udp", udp_dest)
if err != nil {
//fmt.Println("Err, net.ResolveUDPAddr", err)
return
}
conn, err := net.DialUDP("udp", nil, RemoteAddr)
if err != nil {
return
}
udp_message := fmt.Sprintf("<30> %s", *message)
Bytes, _ := conn.Write([]byte(udp_message))
log.Printf("Sent %d Bytes to %s\n", Bytes, udp_dest)
}
func worker(tag int) {
i := 0
for {
worker_message := fmt.Sprintf("Some message from worker%d, loop: %d", tag, i)
log_message(&worker_message)
// do some work..
time.Sleep(300 * time.Second)
i += 1
}
}
In my log_message, everytime it gets called we're calling net.DialUDP which I feel is wasteful. I tried experimenting with global variables &net.UDPConn et al, but could not get to work.
Please show how to achieve/optimize this? There's only one UDP destination, and I'd like the daemon to Dial once at its start, and then just Write as needed.
Thanks!
here's what I got so far:
package main
import (
"fmt"
"log"
"net"
"time"
)
const (
udp_dest = "192.168.1.200:514"
)
var (
myconn *net.UDPConn
)
func main() {
fmt.Println("Hello")
message := "this is a test"
log_message(&message)
go worker(1)
go worker(2)
go worker(3)
go worker(4)
time.Sleep(3009 * time.Second)
}
func log_message(message *string) {
if myconn == nil {
fmt.Println("Setting up myconn!")
RemoteAddr, err := net.ResolveUDPAddr("udp", udp_dest)
if err != nil {
//fmt.Println("Err, net.ResolveUDPAddr", err)
return
}
myconn, err = net.DialUDP("udp", nil, RemoteAddr)
if err != nil {
return
}
}
udp_message := fmt.Sprintf("<30> %s", *message)
Bytes, _ := myconn.Write([]byte(udp_message))
log.Printf("Sent %d Bytes to %s\n", Bytes, udp_dest)
}
func worker(tag int) {
i := 0
for {
worker_message := fmt.Sprintf("Some message from worker%d, loop: %d", tag, i)
log_message(&worker_message)
// do some work..
time.Sleep(10 * time.Second)
i += 1
}
}
You are almost there. Move the setup code to a function and call it before starting the goroutines.
func main() {
if err := setupLog(); err != nil {
log.Fatal(err)
}
fmt.Println("Hello")
... same as before
}
func setupLog() error {
fmt.Println("Setting up myconn!")
RemoteAddr, err := net.ResolveUDPAddr("udp", udp_dest)
if err != nil {
return err
}
myconn, err = net.DialUDP("udp", nil, RemoteAddr)
return err
}
func log_message(message *string) {
udp_message := fmt.Sprintf("<30> %s", *message)
Bytes, _ := myconn.Write([]byte(udp_message))
log.Printf("Sent %d Bytes to %s\n", Bytes, udp_dest)
}
The code in the question does not work because there's a data race on myconn.

How to do websocket portforward with filter or save (in array) some forwarded packet in golang?

i found a ready source code on github for portforwarding in golang
here is the code
package main
import (
"io"
"log"
"net"
)
var localServerHost = "localhost:1020"
var remoteServerHost = "192.168.10.1:1020"
func main() {
ln, err := net.Listen("tcp", localServerHost)
if err != nil {
log.Fatal(err)
}
log.Println("Port forwarding server up and listening on ",
localServerHost)
for {
conn, err := ln.Accept()
if err != nil {
log.Fatal(err)
}
go handleConnection(conn)
}
}
func forward(src, dest net.Conn) {
defer src.Close()
defer dest.Close()
io.Copy(src, dest)
}
func handleConnection(c net.Conn) {
log.Println("Connection from : ", c.RemoteAddr())
remote, err := net.Dial("tcp", remoteServerHost)
if err != nil {
log.Fatal(err)
}
log.Println("Connected to ", remoteServerHost)
go forward(c, remote)
go forward(remote, c)
}
i dont have any idea now how can i put flag on user connections, so then i can read data form user connection and save or filter some of packets depend on connection flag
its a wrong way ?
Edit: This is an answer original question about forwarding and filtering TCP. #ermya has since changed the question to ask about WebSockets. This answer is not relevant to what is now a very different question.
You can filter the stream by interposing a reader or writer in the io.Copy operation. Here's how to interpose a reader:
type myFilter struct {
r io.Reader
}
func (f myFilter) Read(p []byte) (int, error) {
n, err := f.r.Read(p)
// Do something with p[:n]. As an example, the following for loop shows how
// to uppercase ASCII letters. Replace this for loop with the filtering of
// your choice.
for i, b := range p[:n] {
if 'a' <= b && b <= 'z' {
p[i] = b - ('a' - 'A')
}
}
return n, err
}
Filter data from net.Conn by replacing the call to go forward(remote, c) with:
go func() {
defer remote.Close()
defer c.Close()
io.Copy(remote, myFilter{c}) // interpose the filter here
}()
As an aside, the naming in forward is confusing because src is actually the destination and dest is the source. Use this:
func forward(dest, src net.Conn) {
defer src.Close()
defer dest.Close()
io.Copy(dest, src)
}

Sending a CSV file Server to Client

I've just recently started trying to learn Go and I'm trying to write a small server/client application for sending a csv file, from a server to a client. I'm running into an invalid type error when trying to encode a struct into BigEndian binary. My struct seems to already be in a binary format, I'm unsure why I get the following error:
Error writing binary buffer to big endian binary.Write: invalid type *main.DataPack
Also, I'd like to keep the TCP connection open after sending the file, that's why I'm not using io.Copy.
Currently I'm triggering the handling of the file upload through by sending a '\x00' byte:
// Server
package main
import (
"path/filepath"
"fmt"
"io/ioutil"
"net"
"os"
"encoding/binary"
"bytes"
)
type DataPack struct {
Length int64
Contents []byte
}
func main() {
absPath, _ := filepath.Abs("./progs.csv")
data, _ := ioutil.ReadFile(absPath)
fmt.Println("%s",data)
tel, err := net.Listen("tcp", "0.0.0.0:23")
if err != nil {
fmt.Println(err)
return
}
for {
conn, err := tel.Accept()
if err != nil {
break
}
fmt.Println("above filehandler")
go fileHandler(conn)
}
}
func fileHandler(conn net.Conn) {
buf := make([]byte, 0)
buf = append(buf, '\x00')
conn.Write(buf)
absPath, _ := filepath.Abs("./progs.csv")
file, err := os.Open(absPath)
defer file.Close()
fi, err := file.Stat()
if err != nil {
fmt.Println("Fatal error reading file: ", err)
}
fmt.Println("This is the length of the file: ", fi.Size())
data := &DataPack{Length: fi.Size()} // , Contents: }
data.Contents = make([]byte, data.Length)
n, err := file.Read(data.Contents)
if err != nil {
fmt.Println("error reading contents into struct: ", err)
}
fmt.Println("tried to read file contents: ", n)
fmt.Println("DataPack: %+v", data)
buf1 := new(bytes.Buffer)
err = binary.Write(buf1, binary.BigEndian, &data)
if err != nil {
fmt.Println("Error writing binary buffer to big endian ", err)
}
conn.Write(buf1.Bytes())
}
Here is the client
package main
import (
"log"
"fmt"
"net"
"strings"
"strconv"
"bufio"
)
const (
host = "127.0.0.1"
port = 23
)
func main() {
addr := strings.Join([]string{host, strconv.Itoa(port)}, ":")
client := NewClient()
var err error
client.socket, err = net.Dial("tcp", addr)
if err != nil {
log.Println("error setting up socket: %s", err)
}
for {
m := bufio.NewReader(client.socket)
b, err := m.ReadByte()
if err != nil {
fmt.Println("here is the error: ", err)
}
if b == '\x00'{
fmt.Println("about to receive a file!!!")
b, _ := m.ReadByte()
fmt.Println("just got another byte ", b )
}
}
log.Printf("Over")
}
why you get the error
Visit https://golang.org/pkg/encoding/binary/#Write
Write writes the binary representation of data into w. Data must be a fixed-size value or a slice of fixed-size values, or a pointer to such data.
type DataPack struct {
Length int64
Contents []byte
}
Contents isn't a fixed-size value, so you got the invalid type error.
how to solve it
go binary
json
others

Golang TCP File Transfer Gets Stuck In the Middle

I'm having some file transfer issue over TCP in go. The file transfer works sometimes and sometimes it gets stuck in the middle. When it gets stuck, it looks like it is expecting data in the communication channel but there is no data and no error as well. Hence it gets stuck indefinitely. To make thing confusing it shows this behavior for same file i.e for same file it works sometimes and sometimes it doesn't work.
This is how my program works. It'll listen for incoming requests. The requests are in JSON format. Based on request type it'll do different operation. I'm posting the code segment related to file transfer.
server.go
package main
import (
"bufio"
"encoding/json"
"fmt"
_"io"
"net"
"os"
)
const (
COMMAND_RECEIVE_FILE = "TRANSFER_FILE"
COMMAND_EXIT = "EXIT"
CONNECTION_TYPE = "tcp"
CONNECTION_PORT = "3645"
CONNECTION_HOST = ""
BUFFER_SIZE = 1024
)
type Command struct {
Identifier string `json:"identifier"`
Name string `json:"name"`
Size int64 `json:"size"`
}
type Result struct {
Message string `json:"message"`
}
func receiveFile(connection net.Conn, fileName string, fileSize int64) Result {
fmt.Println("Receiving file")
result := Result{Message: ""}
file, err := os.Create(fileName)
if err != nil {
fmt.Println(err)
result.Message = "Error opening file: " + fileName
return result
}
defer file.Close()
fileBuffer := make([]byte, BUFFER_SIZE)
bytesRead := int64(0)
count := 0
for {
if fileSize-bytesRead < int64(BUFFER_SIZE) {
fileBuffer = make([]byte, fileSize-bytesRead)
}
fmt.Println("Reading ", BUFFER_SIZE, " bytes of data")
n, err := connection.Read(fileBuffer)
count++
fmt.Println("Completed reading", n, " bytes of data, count=", count)
file.Write(fileBuffer[0:n])
bytesRead += int64(n)
if err != nil {
result.Message = "File transfer incomplete"
break
}
if bytesRead >= fileSize {
result.Message = "File transfer complete"
break
}
}
file.Chmod(0777)
return result
}
func main() {
ln, err := net.Listen(CONNECTION_TYPE, CONNECTION_HOST + ":"+CONNECTION_PORT)
if err != nil {
fmt.Println("error opening a tcp connection")
}
for {
fmt.Println("waiting for new connection")
conn, err := ln.Accept()
if err != nil {
} else {
var commandStr string
reader := bufio.NewReader(conn)
var exitStatus = 1
for exitStatus == 1 {
fmt.Println("Waiting for new command: ")
line,_,err := reader.ReadLine()
if err != nil {
conn.Close()
exitStatus = 0
break
} else {
fmt.Println("Size read :", len(line))
}
commandStr = string(line)
fmt.Println("CommandStr: ", commandStr)
var msg Command
err = json.Unmarshal([]byte(commandStr), &msg)
if err != nil {
fmt.Println("Error")
conn.Close()
break
}
result := Result{}
fmt.Println("Received new command: ", msg.Identifier)
switch msg.Identifier {
case COMMAND_RECEIVE_FILE:
result = receiveFile(conn, msg.Name, msg.Size)
case COMMAND_EXIT:
exitStatus = 0
conn.Close()
default:
result = Result{Message: "Unrecognized command"}
}
out, _ := json.Marshal(result)
fmt.Fprint(conn, string(out)+"\n")
}
}
}
}
test.go
package main
import (
"bufio"
"encoding/json"
"fmt"
"io"
"log"
"net"
"os"
"strings"
_"time"
)
const (
COMMAND_TRANSFER_FILE = "TRANSFER_FILE"
COMMAND_EXIT = "EXIT"
CONNECTION_TYPE = "tcp"
CONNECTION_PORT = "3645"
CONNECTION_HOST = ""
)
type Command struct {
Identifier string `json:"identifier"`
Name string `json:"name"`
Size int64 `json:"size"`
}
type Result struct {
Message string `json:"message"`
}
func main() {
conn, _ := net.Dial(CONNECTION_TYPE, CONNECTION_HOST + ":" + CONNECTION_PORT)
decoder := json.NewDecoder(conn)
com := Command{}
sourceFileName := ""
destinationFileName := ""
for {
com = Command{}
reader := bufio.NewReader(os.Stdin)
identifier, _ := reader.ReadString('\n')
com.Identifier = strings.TrimSpace(identifier)
switch com.Identifier {
case COMMAND_TRANSFER_FILE:
fmt.Print("Source file name:")
sourceFileName, _ = reader.ReadString('\n')
sourceFileName = strings.TrimSpace(sourceFileName)
fmt.Print("Destination file name:")
destinationFileName, _ = reader.ReadString('\n')
com.Name = strings.TrimSpace(destinationFileName)
file, err := os.Open(sourceFileName)
if err != nil {
log.Fatal(err)
}
defer file.Close()
fileInfo, err := file.Stat()
fileSize := fileInfo.Size()
com.Size = fileSize
case COMMAND_EXIT:
conn.Close()
os.Exit(0)
}
out, _ := json.Marshal(com)
conn.Write([]byte(string(out) + "\n"))
if strings.Compare(com.Identifier, COMMAND_TRANSFER_FILE) == 0 {
file, err := os.Open(sourceFileName)
if err != nil {
log.Fatal(err)
}
defer file.Close()
n, err := io.Copy(conn, file)
if err != nil {
log.Fatal(err)
}
fmt.Println(n, "bytes sent")
}
var msg Result
err := decoder.Decode(&msg)
if err != nil {
fmt.Println(err)
}
fmt.Println(msg)
}
}
I tested it on both Linux and Windows and it shows same behavior on both system. The only thing I can think of is that the sender is faster than the receiver even though I'm running it on the same machine. If that is the case, what will be a best practice to solve it other than the handshaking mechanism.
You can't wrap the net.Conn in a bufio.Reader, then continue to use the net.Conn. The reason your function is blocked is because you left data buffered in the reader, so you won't ever reach the desired message size.
You need to pass the reader to the receiveFile function in order to not lose the buffered data.
You are also ignoring the isPrefix return value from ReadLine. I would follow the documentation and use ReadBytes instead if you're not going to handle all cases from that method.

Resources