Memory management in Go - go

I have a closure where I declare and define local variables:
func writer_factory() func() *net.TCPConn {
response_port := "localhost:8000"
tcpAddr_res, err := net.ResolveTCPAddr("tcp4", response_port)
checkError(err)
var response_writer *net.TCPConn
checkError(err)
return func() *net.TCPConn {
if response_writer == nil{
response_writer, err = net.DialTCP("tcp", nil, tcpAddr_res)
checkError(err)
}
return response_writer
}
}
Now my question is, if I call this writer_factory multiple times will I get a memory leak?
So specifically, will I have a memory leak using writer_factory in this program:
package main
import (
"fmt"
"net"
"os"
"strings"
// "io/ioutil"
)
//fmt.Printf("messages are (1) %q\n", messages)
func main() {
end_of_message_terminator := "||"
beginning_of_next_message := ""
request := make([]byte, 512)
service_port := ":7777"
tcpAddr, err := net.ResolveTCPAddr("tcp4", service_port)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
for {
response_writer := writer_factory()
conn, err := listener.Accept()
if err != nil {
continue
}
read_len, err := conn.Read(request)
if read_len == 0 {
continue
}
request_string := string(request[:read_len])
messages := strings.Split(request_string, end_of_message_terminator)
messages[0] = beginning_of_next_message + messages[0]
if messages[len(messages) - 1] != "" {
beginning_of_next_message = messages[len(messages) - 1]
messages[len(messages) - 1] = ""
}
if len(messages) == 1 {
continue
}
rw := response_writer()
join_channel := make(chan struct{})
for i := 0; i < len(messages); i++ {
go func(i int, rw *net.TCPConn){
respond_to_message(messages[i], rw)
join_channel <- struct{}{}
}(i, rw)
}
go func (){
for i := 0; i < len(messages); i++ {
<- join_channel
}
rw.Close()
}()
conn.Close()
}
}
func writer_factory() func() *net.TCPConn {
response_port := "localhost:8000"
tcpAddr_res, err := net.ResolveTCPAddr("tcp4", response_port)
checkError(err)
var response_writer *net.TCPConn
checkError(err)
return func() *net.TCPConn {
if response_writer == nil{
response_writer, err = net.DialTCP("tcp", nil, tcpAddr_res)
checkError(err)
}
return response_writer
}
}
func respond_to_message(message string, response_writer *net.TCPConn){
message_parameters := strings.Split(message, "|")
//response_writer.Write([]byte("asti de chris"))
for i := range message_parameters {
param_parts := strings.Split(message_parameters[i], "=")
param_name := param_parts[0]
//param_value := param_parts[1]
response_writer.Write([]byte(param_name))
//fmt.Println(string(result))
}
}
func checkError(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "Fatal error: %s", err.Error())
os.Exit(1)
}
}
To exercise this code launch the program, then launch this program netcat -l -p 8000 and then this program printf "asti||" | netcat localhost 7777

You can definitely improve performance by using one connection or (if one connection is not thread save) use a sync.Pool. That would save you memory allocations (and with it garbage collection) and also time wasted on opening all those tcp connections.

Related

go udp requests (packet) loss

i wrote simple server and client for tcp and udp connection
package main
//server.go
import (
"fmt"
"net"
"os"
"os/signal"
"syscall"
)
func main() {
tcp := 0
udp := 0
defer func(o, t *int) {
fmt.Println(*o, *t)
}(&tcp, &udp)
go func() {
l, err := net.ListenTCP("tcp", &net.TCPAddr{
IP: net.ParseIP("0.0.0.0"),
Port: 3000,
})
if err != nil {
panic(err)
}
b := make([]byte, 24)
for {
conn, err := l.Accept()
if err != nil {
continue
}
n, err := conn.Read(b)
if err != nil {
continue
}
r := string(b[:n])
if r == "close" {
conn.Close()
break
} else {
_, err = conn.Write([]byte("pong"))
if err != nil {
continue
}
}
conn.Close()
tcp++
}
l.Close()
}()
go func() {
conn, err := net.ListenUDP("udp", &net.UDPAddr{
IP: net.ParseIP("0.0.0.0"),
Port: 3000,
})
if err != nil {
panic(err)
}
b := make([]byte, 24)
for {
n, addr, err := conn.ReadFromUDP(b)
if err != nil {
continue
}
r := string(b[:n])
if r == "close" {
break
} else {
_, err = conn.WriteToUDP([]byte("pong"), addr)
if err != nil {
continue
}
}
udp++
}
conn.Close()
}()
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
<-signals
}
package main
//client.go
import (
"fmt"
"net"
"os"
"os/signal"
"strconv"
"sync/atomic"
"syscall"
"time"
)
func main() {
t := "tcp"
m := "ping"
c := 1
if len(os.Args) > 1 {
t = os.Args[1]
}
if len(os.Args) > 2 {
m = os.Args[2]
}
if len(os.Args) > 3 {
c, _ = strconv.Atoi(os.Args[3])
}
tcp := int64(0)
udp := int64(0)
defer func(o, t *int64) {
fmt.Println(*o, *t)
}(&tcp, &udp)
if c == 1 {
if t == "tcp" {
addr, err := net.ResolveTCPAddr("tcp", ":3000")
if err != nil {
panic(err)
}
conn, err := net.DialTCP("tcp", nil, addr)
if err != nil {
panic(err)
}
_, err = conn.Write([]byte(m))
if err != nil {
panic(err)
}
tcp++
}
if t == "udp" {
addr, err := net.ResolveUDPAddr("udp", ":3000")
if err != nil {
panic(err)
}
conn, err := net.DialUDP("udp", nil, addr)
if err != nil {
panic(err)
}
_, err = conn.Write([]byte(m))
if err != nil {
panic(err)
}
udp++
}
os.Exit(0)
}
for i := 0; i < c; i++ {
go func() {
a1, err := net.ResolveTCPAddr("tcp", ":3000")
if err != nil {
panic(err)
}
c1, err := net.DialTCP("tcp", nil, a1)
if err != nil {
panic(err)
}
_, err = c1.Write([]byte(m))
if err != nil {
panic(err)
}
buf := make([]byte, 24)
n, err := c1.Read(buf)
if err != nil {
panic(err)
}
if string(buf[:n]) != "pong" {
panic(1)
}
err = c1.Close()
if err != nil {
panic(err)
}
g := atomic.AddInt64(&tcp, 1)
if g % 100 == 0 {
fmt.Println("tcp", g)
time.Sleep(time.Millisecond * 1000)
}
}()
go func() {
a2, err := net.ResolveUDPAddr("udp", ":3000")
if err != nil {
panic(err)
}
c2, err := net.DialUDP("udp", nil, a2)
if err != nil {
panic(err)
}
_, err = c2.Write([]byte(m))
if err != nil {
panic(err)
}
buf := make([]byte, 24)
n, err := c2.Read(buf)
if err != nil {
panic(err)
}
if string(buf[:n]) != "pong" {
panic(1)
}
err = c2.Close()
if err != nil {
panic(err)
}
g := atomic.AddInt64(&udp, 1)
if g % 100 == 0 {
fmt.Println("udp", g)
time.Sleep(time.Millisecond * 1000)
}
}()
}
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
<-signals
}
and get a strange behavior: not all udp requests are sent or handled on many connections.
When i sent 100 or 200 both server and client tells me that all requests worked but from 1000 there is a strange ~5% requests loss on udp for both server and client but no panic.
I know udp allows packet loss but 5% on localhost requests seems like an error.
As we know, the UDP is connectionless, so packet loss may be caused by this nature of UDP. There could be several ways to lower the rate of packet loss.
Slow down the rate of packets sent on the client side
Call SetWriteBuffer to increase the buffer size on the client side and set SetReadBuffer to increase the buffer size on the server side
conn, err := net.DialUDP("udp", nil, addr)
err = conn.SetWriteBuffer(64 * 1024 * 1024)
Check the system network information through netstat -s -udp and get UDP statistics information. You could try to change the default value of rmem_max and rmem_default on the server side. For more details, please refer to 1 and 2

go - Simple worker pool

As a beginner in go and programming in general, I've been coding a port scanner in go with gopacket library and most of the code is done, but I've been running into a problem of spawning too many goroutines and getting 'read ip4 0.0.0.0: i/o timeout' I've done some research and it seems I need to implement a worker pool I've been trying to implement this example 'https://gobyexample.com/worker-pools' as I'm still learning goroutines and channels I've been at it a few days now and can't seem to figure out how to implement the above example correctly in my program can you guys give me some pointers or preferably a code fix example.
package main
import (
"fmt"
"log"
"net"
"time"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
)
// Get preferred outbound ip and port of this machine
func GetOutboundIPPort() (net.IP, int) {
conn, err := net.Dial("udp", "1.1.1.1:80")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
localAddr := conn.LocalAddr().(*net.UDPAddr)
return localAddr.IP, localAddr.Port
}
func ipv4_gen(out chan net.IP) {
ip, ipnet, err := net.ParseCIDR("192.168.0.0/24")
if err != nil {
log.Fatal(err)
}
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
time.Sleep(300 * time.Millisecond)
dstaddrs, err := net.LookupIP(ip.String())
if err != nil {
log.Fatal(err)
}
dstip := dstaddrs[0].To4()
out <- dstip
}
close(out)
}
func inc(ip net.IP) {
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}
func port_scanner(dstip net.IP) {
dstport := layers.TCPPort(80)
srcip, port := GetOutboundIPPort()
srcport := layers.TCPPort(port)
ip := &layers.IPv4{
SrcIP: srcip,
DstIP: dstip,
Protocol: layers.IPProtocolTCP,
}
tcp := &layers.TCP{
SrcPort: srcport,
DstPort: dstport,
Seq: 1105024978,
SYN: true,
Window: 14600,
}
tcp.SetNetworkLayerForChecksum(ip)
buf := gopacket.NewSerializeBuffer()
opts := gopacket.SerializeOptions{
ComputeChecksums: true,
FixLengths: true,
}
if err := gopacket.SerializeLayers(buf, opts, tcp); err != nil {
log.Fatal(err)
}
conn, err := net.ListenPacket("ip4:tcp", "0.0.0.0")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
if _, err := conn.WriteTo(buf.Bytes(), &net.IPAddr{IP: dstip}); err != nil {
log.Fatal(err)
}
// Set deadline so we don't wait forever.
if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
log.Fatal(err)
}
for {
b := make([]byte, 4096)
n, addr, err := conn.ReadFrom(b)
if err != nil {
log.Println("error reading packet: ", err)
return
} else if addr.String() == dstip.String() {
// Decode a packet
packet := gopacket.NewPacket(b[:n], layers.LayerTypeTCP, gopacket.Default)
// Get the TCP layer from this packet
if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
tcp, _ := tcpLayer.(*layers.TCP)
if tcp.DstPort == srcport {
if tcp.SYN && tcp.ACK {
fmt.Printf("Discovered open port %d/tcp on %s\n", dstport, dstip)
}
return
}
}
}
}
}
func worker(id int, ips <-chan net.IP) {
for ip := range ips {
go port_scanner(ip)
}
}
func main() {
ips := make(chan net.IP)
go ipv4_gen(ips)
for w := 1; w <= 10; w++ {
go worker(w, ips)
}
}
You are starting a new goroutine for each piece of work inside the worker so it defeats its purpose.
You need to run the work instead of starting a goroutine inside the worker.

go use fsnotify watch file ischanged not working

I use the library (http://github.com/fsnotify/fsnotify) to monitor the file system.
I m trying to adjust the repository example to match my requirements, but when i do so the program is not working anymore.
I commented the done channel within the ExampleNewWatcher function
done := make(chan bool)
<-done
As a result, now when i run the example, this channel does not output anything anymore.
event, ok := <-watcher.Events
Complete code:
package main
import (
"github.com/fsnotify/fsnotify"
"log"
"os"
"strconv"
"time"
)
func ExampleNewWatcher() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
done := make(chan bool) // if i commen this line the `panic` not norking
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Println("event:", event)
if event.Op&fsnotify.Write == fsnotify.Write {
log.Println("modified file:", event.Name)
}
panic("just for test") // new output this line
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Println("error:", err)
}
}
}()
err = watcher.Add("/tmp/foo")
if err != nil {
log.Fatal(err)
}
<-done // comment this
}
func check(err error) {
if err != nil {
panic(err)
}
}
func main() {
// create the test file
var err error
file, err := os.Create("/tmp/foo")
check(err)
_, err = file.Write([]byte("hello world"))
check(err)
stopchan := make(chan struct{})
// test change file
go func() {
for i := 0; i < 10; i++ {
file1, err := os.OpenFile("/tmp/foo", os.O_RDWR, 0644)
check(err)
d := strconv.Itoa(i) + "hello world"
_, err = file1.Write([]byte(d))
check(err)
err = file1.Close()
check(err)
time.Sleep(2 * time.Second) // wait the context writed to the file
}
}()
ExampleNewWatcher() // monitor file
stopchan <- struct{}{}
}

Client stuck when trying to read with io.CopyN() in golang

I am trying to make TCP server for transferring files. I am suing io.CopyN for reading and writing. From server side, I am sending files to client so from server side, it sends perfectly all bytes but Client side after reading a couple of 1000000 bytes it stuck. sometimes it works fine and sometimes it gets stuck. I am using 300 MB pdf to test. Any help, code, and output is like below.
server
package main
import (
"fmt"
"io"
"log"
"net"
"os"
"strconv"
"strings"
)
func main() {
ls, err := net.Listen("tcp", ":1234")
errFunc(err)
defer ls.Close()
conn, _ := ls.Accept()
defer conn.Close()
for {
file, err := os.Open(strings.TrimSpace("./" + "Mag" + ".pdf"))
errFunc(err)
defer file.Close()
fileInfo, err := file.Stat()
errFunc(err)
size := fileInfo.Size()
numberOfTime := size / 1000000
leftByte := size - numberOfTime*1000000
numberOfTimeString := strconv.Itoa(int(numberOfTime))
leftByteString := strconv.Itoa(int(leftByte))
fmt.Println("1000000 times : ", numberOfTimeString)
fmt.Println("Left Bytes : ", leftByteString)
_, err = fmt.Fprintf(conn, numberOfTimeString+"\n")
errFunc(err)
_, err = fmt.Fprintf(conn, leftByteString+"\n")
errFunc(err)
fileWriter := io.Writer(conn)
for i := 0; i < int(numberOfTime); i++ {
n, err := io.CopyN(conn, file, 1000000)
if i >= 30 {
fmt.Println(err, n)
}
}
n, err := io.CopyN(fileWriter, file, leftByte+1)
if err == io.EOF {
fmt.Println(err, n)
}
fmt.Printf("Succefully bytes sent : %v \n\n\n\n\n", n)
file.Close()
}
}
func errFunc(err error) {
if err != nil {
log.Fatal(err)
}
}
client
package main
import (
"bufio"
"fmt"
"io"
"net"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
)
func main() {
c := make(chan os.Signal, 15)
signal.Notify(c, syscall.SIGINT)
go func() {
for {
s := <-c
switch s {
case syscall.SIGINT:
os.Exit(1)
}
}
}()
conn, _ := net.Dial("tcp", ":1234")
defer conn.Close()
connReadWrite := bufio.NewReader(io.Reader(conn))
var i int
var filename string
for {
i++
nu := strconv.Itoa(i)
filename = "image" + nu + ".pdf"
file, err := os.Create(filename)
defer file.Close()
numberOfTimeString, err := connReadWrite.ReadString('\n')
if err != nil {
fmt.Println(err)
}
println("1000000 times :", numberOfTimeString)
numberOfTimeString = strings.TrimSuffix(numberOfTimeString, "\n")
numberOfTime, err := strconv.Atoi(numberOfTimeString)
if err != nil {
fmt.Println(err)
}
leftByteString, err := connReadWrite.ReadString('\n')
if err != nil {
println(err)
}
println("Left Bytes :", leftByteString)
leftByteString = strings.TrimSuffix(leftByteString, "\n")
leftByte, err := strconv.Atoi(leftByteString)
if err != nil {
panic(err)
}
fmt.Println("After convert in Num :", numberOfTime, leftByte)
newFileWriter := io.Writer(file)
newFileReader := io.Reader(conn)
for i := 0; i < numberOfTime; i++ {
n, err := io.CopyN(newFileWriter, newFileReader, 1000000)
if i >= 30 {
errFun(err, n)
}
}
n, err := io.CopyN(newFileWriter, newFileReader, int64(leftByte))
errFun(err, n)
fmt.Printf("sucessfully Transfered ---> \n\n\n\n\n\n")
}
}
func errFun(err error, n int64) {
if err == io.EOF {
fmt.Println("End of file : ", n)
return
} else if n == 0 {
fmt.Println("n is : ", n)
return
} else if err != nil {
fmt.Println(err)
return
}
fmt.Println(err, " : ", n)
}
input/output
from server side first we are sending number of bytes it need to readand then client side it gets a number of bytes it needs to read and then I am sending the file and then it read. In the picture, I was able to send one-time second time it got stuck sometimes it stuck first time too.I am able to send number of byte from server side second time too but as you can see it don't read that numeber, it read something "%PDF..." and it even don't print "100000 times : " correctly it prints "%???00 times :" I just don’t understand this
enter image description here
I believe the issue is that you're using a bytes.Buffer in the client:
connReadWrite := bufio.NewReader(io.Reader(conn))
But you aren't using it later with the CopyN:
newFileWriter := io.Writer(file)
newFileReader := io.Reader(conn)
for i := 0; i < numberOfTime; i++ {
_, err := io.CopyN(newFileWriter, newFileReader, 1000000)
if err != nil {
log.Fatalln(err)
}
}
Using:
newFileWriter := io.Writer(file)
for i := 0; i < numberOfTime; i++ {
_, err := io.CopyN(file, connReadWrite, 1000000)
if err != nil {
log.Fatalln(err)
}
}
May fix it.
If you have control over the protocol you are using to send the file, I recommend doing something simpler. For example using the big-endian int64 length prefix.
Send:
func sendFile(name string, conn net.Conn) error {
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return err
}
sz := fi.Size()
buf := bufio.NewWriter(conn)
err = binary.Write(buf, binary.BigEndian, sz)
if err != nil {
return err
}
_, err = io.CopyN(buf, f, sz)
if err != nil {
return err
}
return buf.Flush()
}
Receive:
func recvFile(name string, conn net.Conn) error {
f, err := os.Create(name)
if err != nil {
return err
}
defer f.Close()
buf := bufio.NewReader(conn)
var sz int64
err = binary.Read(buf, binary.BigEndian, &sz)
if err != nil {
return err
}
_, err = io.CopyN(f, buf, sz)
if err != nil {
return err
}
return nil
}

Golang TCPConn Gob Communication

I'm having issues with the gob protocol (or maybe networking in general, where my knowledge is weak), and I don't understand why the following code does not work properly. It is just supposed to be a simple example of maintaining an open TCP connection, and sending multiple gobs through it. The code will send, and receive, but often corrupts its data.
Thank you in advance.
package main
import (
"encoding/gob"
"fmt"
"net"
"strconv"
"time"
)
type Message struct {
Msg string
}
func main() {
gob.Register(new(Message))
clientAddr, err := net.ResolveTCPAddr("tcp", "localhost:12346")
if err != nil {
fmt.Println(err)
}
serverAddr, err := net.ResolveTCPAddr("tcp", "localhost:12345")
if err != nil {
fmt.Println(err)
}
serverListener, err := net.ListenTCP("tcp", serverAddr)
if err != nil {
fmt.Println(err)
}
conn, err := net.DialTCP("tcp", clientAddr, serverAddr)
if err != nil {
fmt.Println(err)
}
serverConn, err := serverListener.AcceptTCP()
if err != nil {
fmt.Println(err)
}
done := false
go func() {
for !done {
recieveMessage(serverConn)
}
}()
for i := 1; i < 1000; i++ {
sent := Message{strconv.Itoa(i)}
sendMessage(sent, conn)
}
time.Sleep(time.Second)
done = true
}
func sendMessage(msg Message, conn *net.TCPConn) {
enc := gob.NewEncoder(conn)
err := enc.Encode(msg)
if err != nil {
fmt.Println(err)
}
}
func recieveMessage(conn *net.TCPConn) {
msg := new(Message)
dec := gob.NewDecoder(conn) // Will read from network.
err := dec.Decode(msg)
if err != nil {
fmt.Println(err)
}
fmt.Println("Client recieved:", msg.Msg)
}
The problem is that the decoder can buffer data from the next message. When this happens, the next new decoder starts in the middle of a message. The fix is to use a single encoder and decoder.
func main() {
...
dec := gob.NewDecoder(conn) // Will read from network.
enc := gob.NewEncoder(serverConn)
go func() {
for !done {
recieveMessage(dec)
}
}()
for i := 1; i < 1000; i++ {
sent := Message{strconv.Itoa(i)}
sendMessage(sent, enc)
}
...
}
func sendMessage(msg Message, enc *gob.Encoder) {
err := enc.Encode(msg)
if err != nil {
fmt.Println(err)
}
}
func recieveMessage(dec *gob.Decoder) {
msg := new(Message)
err := dec.Decode(msg)
if err != nil {
fmt.Println(err)
}
fmt.Println("Client recieved:", msg.Msg)
}
Run it in the playground

Resources