How to find server real IP - go

How do I find the public IP address of the machine or server my program is running on?
Like when the program is executed it detect the public of the server and print for e.g.running at 123.45.67.89

The short answer is that there's no method guaranteed to return your "public" ip address.
The first question is, what is your public ip address? The address of your machine, as seen by the system to which you are connecting, may vary depending on how your local Internet service is configured and on the service to which you're connecting:
As I mentioned in a comment, in a typical home environment your machine doesn't have a public ip address. The public address is hosted by your router.
If you're accessing a service through a proxy or vpn, your machine's address may be entirely different from when you're directly connecting to a service.
On a system with multiple interfaces, the origin address selected may depend upon the address to which you are connecting: different addresses may have different routes.
You can try using a service like http://icanhazip.com/ to try to determine your public ip. This will be correct in many but not all situations.

public IP address is a vague notion, in practice, it might or might not be a static address. What do you know about that ? It is just a endpoint valid for a certain amount of time, which depends on many factors like which interface was used to issue the query.
We can use the mainline bittorrent dht to give us some indications.
The Go language provides the cool dht package written by anacrolix.
When querying nodes with a find_peers verb we receive a packet containing the remote ip address the peer has associated with our query. This is described in bep10.
If an UDP connection is not a good option, you might opt for a query to bittorent trackers as described in bep24
Consider that peers might be malicious, thus the more results, the better.
Below program outputs the list of external network addresses associated with the computer initiating the query from the POV of the cohort of nodes queried.
Addresses are scored by the numbers of response.
read also https://www.bittorrent.org/beps/bep_0005.html
found 9 bootstrap peers
found 6 peers
4 [2001:861:51c5:xxx:40d1:8061:1fe0:xxx]:9090
2 81.96.42.191:9090
4 peers told us that we were using [2001:861:51c5:xxx:40d1:8061:1fe0:xxx]:9090, we can infer this is ipv6.
2 of them told we were using 81.96.42.191:9090, the ipv4 interface.
package main
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net"
"os"
"sort"
"sync"
"time"
"github.com/anacrolix/dht"
"github.com/anacrolix/dht/krpc"
"github.com/anacrolix/torrent/bencode"
)
var maxTimeout = time.Second * 5
func main() {
b, _ := ioutil.ReadFile("db.json")
var rawAddrs []string
json.Unmarshal(b, &rawAddrs)
defer func() {
if len(rawAddrs) < 1 {
return
}
if len(rawAddrs) > 30 {
rawAddrs = rawAddrs[:30]
}
buf, err := json.Marshal(rawAddrs)
if err != nil {
panic(err)
}
err = ioutil.WriteFile("db.json", buf, os.ModePerm)
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "%v peers recorded\n", len(rawAddrs))
}()
bootstrap, err := parseAddrs(rawAddrs)
if err != nil {
bootstrap, err = globalBootstrapAddrs()
if err != nil {
panic(err)
}
}
findPeers := []byte(`d1:ad2:id20:abcdefghij01234567899:info_hash20:mnopqrstuvwxyz123456e1:q9:get_peers1:t2:aa1:y1:qe`)
local, err := net.ResolveUDPAddr("udp", "0.0.0.0:9090")
if err != nil {
panic(err)
}
ln, err := net.ListenUDP("udp", local)
if err != nil {
panic(err)
}
addrscores := map[string]int{}
var drain drain
defer drain.Wait()
fmt.Fprintf(os.Stderr, "found %v bootstrap peers\n", len(bootstrap))
res, errs := readResponses(ln, len(bootstrap), sendQuery(ln, bootstrap, findPeers))
drain.Errors(errs)
peers := []net.Addr{}
for d := range res {
if isValidAddr(d.IP.UDP()) {
addrscores[d.IP.String()]++
d.R.ForAllNodes(func(arg1 krpc.NodeInfo) {
peers = append(peers, arg1.Addr.UDP())
})
}
}
if len(peers) > 0 {
fmt.Fprintf(os.Stderr, "found %v peers\n", len(peers))
res, errs = readResponses(ln, len(peers), sendQuery(ln, peers, findPeers))
drain.Errors(errs)
for d := range res {
if isValidAddr(d.IP.UDP()) {
addrscores[d.IP.String()]++
}
}
}
for _, peer := range peers {
if isValidAddr(peer) {
rawAddrs = append(rawAddrs, peer.String())
}
}
addrs := make([]string, 0, len(addrscores))
for addr := range addrscores {
addrs = append(addrs, addr)
}
sort.Slice(addrs, func(i int, j int) bool {
return addrscores[addrs[i]] > addrscores[addrs[j]]
})
for _, addr := range addrs {
fmt.Printf("%-4v %v\n", addrscores[addr], addr)
}
}
type drain struct{ sync.WaitGroup }
func (d *drain) Errors(errs <-chan error) {
d.Add(1)
go func() {
defer d.Done()
for err := range errs {
fmt.Fprintln(os.Stderr, err)
}
}()
}
func parseAddrs(rawAddrs []string) (addrs []net.Addr, err error) {
for _, s := range rawAddrs {
host, port, err := net.SplitHostPort(s)
if err != nil {
panic(err)
}
ua, err := net.ResolveUDPAddr("udp", net.JoinHostPort(host, port))
if err != nil {
log.Printf("error resolving %q: %v", host, err)
continue
}
addrs = append(addrs, ua)
}
if len(addrs) == 0 {
err = errors.New("nothing resolved")
}
return
}
func globalBootstrapAddrs() (addrs []net.Addr, err error) {
bootstrap, err := dht.GlobalBootstrapAddrs("udp")
if err != nil {
return nil, err
}
for _, b := range bootstrap {
addrs = append(addrs, b.Raw())
}
return
}
func isValidAddr(addr net.Addr) bool { // so weird guys.
return addr.String() != "<nil>" && addr.String() != ":0"
}
func sendQuery(ln *net.UDPConn, peers []net.Addr, query []byte) chan error {
errs := make(chan error)
for _, addr := range peers {
go func(addr net.Addr) {
_, err := ln.WriteTo(query, addr)
if err != nil {
errs <- addressedError{Op: "send", error: err, Addr: addr}
}
}(addr)
}
return errs
}
func readResponses(ln *net.UDPConn, count int, errs chan error) (<-chan krpc.Msg, <-chan error) {
data := make(chan krpc.Msg)
var wg sync.WaitGroup
for i := 0; i < count; i++ {
wg.Add(1)
go func() {
defer wg.Done()
buf := make([]byte, 1000)
ln.SetReadDeadline(time.Now().Add(maxTimeout))
n, remoteAddr, err := ln.ReadFromUDP(buf)
if err != nil {
errs <- addressedError{Op: "rcv", error: err, Addr: remoteAddr}
return
}
var m krpc.Msg
err = bencode.Unmarshal(buf[:n], &m)
if err != nil {
errs <- addressedError{Op: "rcv", error: err, Addr: remoteAddr}
return
}
data <- m
}()
}
go func() {
wg.Wait()
close(errs)
close(data)
}()
return data, errs
}
type addressedError struct {
error
Op string
Addr net.Addr
}
func (a addressedError) Error() string {
if !isValidAddr(a.Addr) {
return fmt.Sprintf("%-5v %v", a.Op, a.error.Error())
}
return fmt.Sprintf("%-5v %v: %v", a.Op, a.Addr.String(), a.error.Error())
}

Related

Attempt to do TLS over UDP: wsarecv error about large message

I want get tls conn base on udp, just like:
package main
import (
"crypto/tls"
"fmt"
"io"
"net"
"time"
)
func handleConn(conn net.Conn) {
defer conn.Close()
var buf []byte = make([]byte, 2000)
for {
if n, err := conn.Read(buf); err != nil && err != io.EOF {
panic(err)
} else {
conn.Write(buf[:n])
}
}
}
func server() {
cert, err := tls.LoadX509KeyPair("./serve.cert.pem", "./serve.key.pem")
if err != nil {
panic(err)
}
config := &tls.Config{Certificates: []tls.Certificate{cert}}
conn, err := net.DialUDP("udp", &net.UDPAddr{Port: 19986}, &net.UDPAddr{Port: 19987})
if err != nil {
panic(err)
}
tconn := tls.Server(conn, config)
defer tconn.Close()
if err = tconn.Handshake(); err != nil {
panic(err)
}
handleConn(tconn)
}
func main() {
go server()
time.Sleep(time.Second)
client()
}
func client() {
conf := &tls.Config{
InsecureSkipVerify: true,
}
conn, err := net.DialUDP("udp", &net.UDPAddr{Port: 19987}, &net.UDPAddr{Port: 19986})
if err != nil {
panic(err)
}
tconn := tls.Client(conn, conf)
defer tconn.Close()
if err = tconn.Handshake(); err != nil {
panic(err)
}
_, err = conn.Write([]byte("hello"))
if err != nil {
panic(err)
}
buf := make([]byte, 100)
n, err := conn.Read(buf)
if err != nil {
panic(err)
}
fmt.Println(string(buf[:n]))
}
panic:
panic: read udp 127.0.0.1:19987->127.0.0.1:19986: wsarecv: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself.
goroutine 1 [running]:
main.client()
D:/OneDrive/code/go/ctest/main.go:65 +0x2d1
main.main()
D:/OneDrive/code/go/ctest/main.go:49 +0x34
exit status 2
Appendix:
serve.cert.pem
-----BEGIN CERTIFICATE-----
MIIBbjCCARSgAwIBAgIRAI+jBYEYS5aBXDUedBt7PKYwCgYIKoZIzj0EAwIwEjEQ
MA4GA1UEChMHQWNtZSBDbzAeFw0yMjAxMDkxNzQxMjBaFw0yMzAxMDkxNzQxMjBa
MBIxEDAOBgNVBAoTB0FjbWUgQ28wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQy
l1/gWhTxZ3rS/XJOMLHhmkQp64EtPrEgq9SjKDpWBZQC+kNZdM5xzJrv3bLqcyOS
JywZfEpTZzW7sxko4maBo0swSTAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYI
KwYBBQUHAwEwDAYDVR0TAQH/BAIwADAUBgNVHREEDTALgglsb2NhbGhvc3QwCgYI
KoZIzj0EAwIDSAAwRQIhAICxMC8o603GwL3bf42EXrtPP5/LtEIc/hjdJpilqc3b
AiBTEdrE+/oCgUjsxV2RFj1+42CTGtcav4sJyCPjme0N/w==
-----END CERTIFICATE-----
serve.key.pem
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgmH+BleetLN1fK0dy
JpedWG8C2yxtb7gEEAwvdwXf6FihRANCAAQyl1/gWhTxZ3rS/XJOMLHhmkQp64Et
PrEgq9SjKDpWBZQC+kNZdM5xzJrv3bLqcyOSJywZfEpTZzW7sxko4maB
-----END PRIVATE KEY-----
I try set udp conn's buff, useless!
The certificate only 558 Bytes, but the UDP has a capacity of 65536 Bytes. then, I debug found: in crypt/tls/conn.go>readRecordOrCCS c.readFromUntil(c.conn, recordHeaderLen), the recordHeaderLen is constant 5, it add bytes.MinRead; so the reader buff's len is 517. Actually data size about 700 Bytes.
So? What should I do?
You cannot sensibly expect to make the thing work: TLS (and SSL) is designed to be carried out using the application layer of another protocol.
What this means in simpler terms is that while TLS is oblivious to what particular protocol transports its bytes, as a client of the application layer, it expects two properties from the underlying stack: it transports arbitrary opaque streams of data, ensuring robustness and in-order delivery.
In the TCP/IP world, this means TCP because UDP does not provide neither of the properties TLS expects.
Still, some solutions do successfully employ TLS over UDP, with OpenVPN being a prime example of this—it uses TLS for handshake and initial key exchange while having UDP as its default, and recommended, transport protocol,—but all of them have a layer implemented over UDP which "arms" it with the properties typically found in TCP: such layers deal with dropped, reordered, duplicated and corrupted UDP frames, "exporting" to the upper layer a protocol which is OK to use for application layer.
This basically means than if you're after using TLS over UDP, you first need to implemet a custom type having the net.Conn interface which would use a UDP association to shovel the bytes back and forth and "export" to its client a connection which has the properties of robustness and in-order delivery—just as net.TCPConn does.
I GOT IT!
just make udp conn become to stream io.
package main
import (
"bufio"
"crypto/tls"
"fmt"
"io"
"net"
"time"
)
func handleConn(conn net.Conn) {
defer conn.Close()
var buf []byte = make([]byte, 2000)
for {
if n, err := conn.Read(buf); err != nil && err != io.EOF {
panic(err)
} else {
fmt.Println(string(buf[:n]))
conn.Write(append([]byte("got: "), buf[:n]...))
}
}
}
func server() {
cert, err := tls.LoadX509KeyPair("./serve.cert.pem", "./serve.key.pem")
if err != nil {
panic(err)
}
config := &tls.Config{Certificates: []tls.Certificate{cert}}
conn, err := net.DialUDP("udp", &net.UDPAddr{Port: 19986}, &net.UDPAddr{Port: 19987})
if err != nil {
panic(err)
}
sconn := NewSconn(conn)
tconn := tls.Server(sconn, config)
defer tconn.Close()
handleConn(tconn)
}
func main() {
go server()
time.Sleep(time.Second)
client()
}
func client() {
conf := &tls.Config{
InsecureSkipVerify: true,
CipherSuites: []uint16{
tls.TLS_AES_128_GCM_SHA256,
},
}
conn, err := net.DialUDP("udp", &net.UDPAddr{Port: 19987}, &net.UDPAddr{Port: 19986})
if err != nil {
panic(err)
}
sconn := NewSconn(conn)
tconn := tls.Client(sconn, conf)
defer tconn.Close()
_, err = tconn.Write([]byte("hello"))
if err != nil {
panic(err)
}
buf := make([]byte, 100)
n, err := tconn.Read(buf)
if err != nil {
panic(err)
}
fmt.Println(string(buf[:n]))
}
//
//
//
//
// --------------------------------------------------
type sconn struct {
conn net.Conn
buf *bytes.Buffer
rLen int
lock sync.Mutex
}
func NewSconn(conn net.Conn) *sconn {
return &sconn{
conn: conn,
buf: bytes.NewBuffer(nil),
rLen: 2000,
lock: sync.Mutex{},
}
}
func (s *sconn) Read(b []byte) (n int, err error) {
s.lock.Lock()
defer s.lock.Unlock()
if s.buf.Len() != 0 {
n, err = s.buf.Read(b)
} else {
s.push(&s.rLen)
n, err = s.buf.Read(b)
}
return n, err
}
// push push UDP package to buff
func (s *sconn) push(rLen *int) (err error) {
defer func() {
if e := recover(); e != nil {
if *rLen < 65536 {
*rLen = *rLen + 2000
s.push(rLen)
} else {
err = errors.New(fmt.Sprintln(e))
}
}
}()
var tmp []byte = make([]byte, *rLen)
if n, err := s.conn.Read(tmp); err != nil {
return err
} else {
_, err := s.buf.Write(tmp[:n])
return err
}
}
func (s *sconn) Write(b []byte) (n int, err error) {
for i := 0; i < len(b); i = i + 512 {
j := i + 512
if j > len(b) {
j = len(b)
}
if n, err = s.conn.Write(b[i:j]); err != nil {
return n, err
}
}
return len(b), nil
}
func (s *sconn) Close() error { return nil } // only close TLS conn, don't close UDP conn
func (s *sconn) LocalAddr() net.Addr { return s.conn.LocalAddr() }
func (s *sconn) RemoteAddr() net.Addr { return s.conn.RemoteAddr() }
func (s *sconn) SetDeadline(t time.Time) error { return s.conn.SetDeadline(t) }
func (s *sconn) SetReadDeadline(t time.Time) error { return s.conn.SetReadDeadline(t) }
func (s *sconn) SetWriteDeadline(t time.Time) error { return s.conn.SetWriteDeadline(t) }

go - Simple worker pool

As a beginner in go and programming in general, I've been coding a port scanner in go with gopacket library and most of the code is done, but I've been running into a problem of spawning too many goroutines and getting 'read ip4 0.0.0.0: i/o timeout' I've done some research and it seems I need to implement a worker pool I've been trying to implement this example 'https://gobyexample.com/worker-pools' as I'm still learning goroutines and channels I've been at it a few days now and can't seem to figure out how to implement the above example correctly in my program can you guys give me some pointers or preferably a code fix example.
package main
import (
"fmt"
"log"
"net"
"time"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
)
// Get preferred outbound ip and port of this machine
func GetOutboundIPPort() (net.IP, int) {
conn, err := net.Dial("udp", "1.1.1.1:80")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
localAddr := conn.LocalAddr().(*net.UDPAddr)
return localAddr.IP, localAddr.Port
}
func ipv4_gen(out chan net.IP) {
ip, ipnet, err := net.ParseCIDR("192.168.0.0/24")
if err != nil {
log.Fatal(err)
}
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
time.Sleep(300 * time.Millisecond)
dstaddrs, err := net.LookupIP(ip.String())
if err != nil {
log.Fatal(err)
}
dstip := dstaddrs[0].To4()
out <- dstip
}
close(out)
}
func inc(ip net.IP) {
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}
func port_scanner(dstip net.IP) {
dstport := layers.TCPPort(80)
srcip, port := GetOutboundIPPort()
srcport := layers.TCPPort(port)
ip := &layers.IPv4{
SrcIP: srcip,
DstIP: dstip,
Protocol: layers.IPProtocolTCP,
}
tcp := &layers.TCP{
SrcPort: srcport,
DstPort: dstport,
Seq: 1105024978,
SYN: true,
Window: 14600,
}
tcp.SetNetworkLayerForChecksum(ip)
buf := gopacket.NewSerializeBuffer()
opts := gopacket.SerializeOptions{
ComputeChecksums: true,
FixLengths: true,
}
if err := gopacket.SerializeLayers(buf, opts, tcp); err != nil {
log.Fatal(err)
}
conn, err := net.ListenPacket("ip4:tcp", "0.0.0.0")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
if _, err := conn.WriteTo(buf.Bytes(), &net.IPAddr{IP: dstip}); err != nil {
log.Fatal(err)
}
// Set deadline so we don't wait forever.
if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
log.Fatal(err)
}
for {
b := make([]byte, 4096)
n, addr, err := conn.ReadFrom(b)
if err != nil {
log.Println("error reading packet: ", err)
return
} else if addr.String() == dstip.String() {
// Decode a packet
packet := gopacket.NewPacket(b[:n], layers.LayerTypeTCP, gopacket.Default)
// Get the TCP layer from this packet
if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
tcp, _ := tcpLayer.(*layers.TCP)
if tcp.DstPort == srcport {
if tcp.SYN && tcp.ACK {
fmt.Printf("Discovered open port %d/tcp on %s\n", dstport, dstip)
}
return
}
}
}
}
}
func worker(id int, ips <-chan net.IP) {
for ip := range ips {
go port_scanner(ip)
}
}
func main() {
ips := make(chan net.IP)
go ipv4_gen(ips)
for w := 1; w <= 10; w++ {
go worker(w, ips)
}
}
You are starting a new goroutine for each piece of work inside the worker so it defeats its purpose.
You need to run the work instead of starting a goroutine inside the worker.

How to cache a TCP reverse proxy data transmission?

I've accomplished implementing TCP reverse proxy in GoLang. But unfortunately couldn't come up with implementing caching to a TCP reverse proxy. Is it possible to do so, if yes, is there any resource out there? Is caching possible on a TCP (Transport Layer of Network)?
Here's the simple TCP reverse proxy in Golang.
package main
import (
"io"
"log"
"net"
)
//Proxy struct
type Proxy struct {
laddr, raddr *net.TCPAddr
lconn, rconn io.ReadWriteCloser
errorSignal chan bool
}
// New Create a new Proxy instance.
func New(lconn *net.TCPConn, laddr, raddr *net.TCPAddr) *Proxy {
return &Proxy{
lconn: lconn,
laddr: laddr,
raddr: raddr,
errorSignal: make(chan bool),
}
}
//TCPAddressResolver resolves an address and returns to a struct having ip and port.
func TCPAddressResolver(addr string) (tcpAddress *net.TCPAddr, err error) {
tcpAddress, err = net.ResolveTCPAddr("tcp", addr)
return
}
func main() {
listenerAddress, err := TCPAddressResolver(":8080")
if err != nil {
log.Fatalf("Failed to resolve local address: %v", err)
}
remoteAddress, err := TCPAddressResolver(":3000")
if err != nil {
log.Fatalf("Failed to resolve remote address: %v", err)
}
listener, err := net.ListenTCP("tcp", listenerAddress)
if err != nil {
log.Fatalf("Failed to open local port to listen: %v", err)
}
log.Printf("Simple Proxy started on: %d and forwards to port %d", listenerAddress.Port, remoteAddress.Port)
for {
conn, err := listener.AcceptTCP()
if err != nil {
log.Fatalf("Failed to accept connection: %v", err)
continue
}
var p *Proxy
// HTTP is a stateless protocol thus a proxy needs to reinitiate the new next incoming call (conn)
// each time it finishes handling the previous one.
p = New(conn, listenerAddress, remoteAddress)
p.Start()
}
}
//Start initiates transmission of data to and from the remote to client side.
func (p *Proxy) Start() {
defer p.lconn.Close()
var err error
p.rconn, err = net.DialTCP("tcp", nil, p.raddr)
if err != nil {
log.Fatalf("Remote connection failure: %v", err)
}
defer p.rconn.Close()
go p.CopySrcDst(p.lconn, p.rconn)
go p.CopySrcDst(p.rconn, p.lconn)
//Wait for everything to close -- This one blocks the routine.
<-p.errorSignal
log.Printf("Closing Start routine \n")
}
func (p *Proxy) err(err error) {
if err != io.EOF {
log.Printf("Warning: %v: Setting error signal to true", err)
}
p.errorSignal <- true
}
//CopySrcDst copies data from src to dest
func (p *Proxy) CopySrcDst(src, dst io.ReadWriteCloser) {
buff := make([]byte, 1024)
for {
n, err := src.Read(buff)
if err != nil {
// Reading error.
p.err(err)
return
}
dataFromBuffer := buff[:n]
n, err = dst.Write(dataFromBuffer)
if err != nil {
// Writing error.
p.err(err)
return
}
}
}
You are asking how to save data read from an io.Reader. That's different from caching.
The easiest approach is to tee the reader into a buffer.
While you are at it, you might as well use io.Copy instead of the similar code in the question. The code in the question does not handle the case when read returns n > 0 and a non-nil error.
Use an error group to coordinate waiting for the goroutines and collecting error status.
var g errgroup.Group
var rbuf, lbuf bytes.Buffer
g.Go(func() error {
_, err := io.Copy(lconn, io.TeeReader(p.rconn, &rbuf))
return err
})
g.Go(func() error {
_, err := io.Copy(rconn, io.TeeReader(p.lconn, &lbuf))
return err
})
if err := g.Wait(); err != nil {
// handle error
}
// rbuf and lbuf have the contents of the two streams.
The name of the programming language is "Go", not "Golang" or "GoLang".

Simple server client communication not working

This seemingly simple example is not working as expected and I feel bad for asking but here goes:
There's a client that retries connecting to the server, sends a message, and then waits for a response:
func client() {
var conn net.Conn
var err error
// retry server until it is up
for {
conn, err = net.Dial("tcp", ":8081")
if err == nil {
break
}
log.Println(err)
time.Sleep(time.Second)
}
// write to server
_, err = conn.Write([]byte("request"))
if err != nil {
log.Println(err)
return
}
// block & read from server
var buf []byte
n, err := conn.Read(buf)
if err != nil {
log.Println(err)
return
}
log.Printf("From server: %s\n", buf[:n])
}
It connects to a server which for each connection, reads and interprets the sent data, and sends a response if needed:
func server() {
ln, _ := net.Listen("tcp", ":8081")
for {
conn, _ := ln.Accept()
go handleConn(conn)
}
}
func handleConn(conn net.Conn) {
var buf []byte
n, err := conn.Read(buf)
if err != nil {
return
}
log.Printf("Server got: %s\n", buf)
if string(buf[:n]) == "request" {
_, _ = conn.Write([]byte("response"))
}
}
All driven by the main function:
func main() {
go client()
server()
}
Error handling is omitted for brevity. The expected behavior is that the client will connect to the server and send the message "request" and then block on the read. The server receives "request" and sends the message "response" back to the same connection. The client unblock, prints the received message and exits. Instead, when the program is run, the following is printed:
2019/09/01 22:24:02 From server:
2019/09/01 22:24:02 Server got:
Suggesting that no data was exchanged, and that the client did not block.
The looping in client is strange!
The looping not make sense if read/write is out.
But the error is only this:
//var buf []byte <--- this read 0 bytes
buf := make([]byte, 1024)
n, err := conn.Read(buf)
A proposal for you:
package main
import (
"log"
"net"
"time"
)
func client() {
var conn net.Conn
var err error
// retry server until it is up
for {
log.Printf("Connecting...")
conn, err = net.Dial("tcp", ":8082")
if err != nil {
log.Println(err)
break
}
time.Sleep(time.Second)
// write to server
log.Printf("Writing...")
_, err = conn.Write([]byte("request"))
if err != nil {
log.Println(err)
return
}
// block & read from server
log.Printf("Reading...")
var buf []byte
n, err := conn.Read(buf)
if err != nil {
log.Println(err)
return
}
log.Printf("From server: %s\n", buf[:n])
}
}
func server() {
ln, _ := net.Listen("tcp", ":8082")
for {
conn, _ := ln.Accept()
go handleConn(conn)
}
}
func handleConn(conn net.Conn) {
buf := make([]byte, 1024)
n, err := conn.Read(buf)
if err != nil {
return
}
log.Printf("Server got: [%d bytes] %s\n", n, buf)
if string(buf[:n]) == "request" {
_, _ = conn.Write([]byte("response"))
}
conn.Close()
}
func main() {
go client()
server()
}

Why does this Golang app use more memory the longer it runs?

I made this to monitor a few websites and notify me if one of them goes down. I'm testing it on just two urls. When it starts it uses about 5mb of memory (I checked with systemctl status monitor). After 40 minutes, it's using 7.4mb. After 8 hours, it uses over 50mb of memory. Why is it doing this? Is this called a memory leak?
package main
import (
"fmt"
"io/ioutil"
"net/http"
"os"
"sync"
"time"
"monitor/utils/slack"
"gopkg.in/yaml.v2"
)
var config struct {
Frequency int
Urls []string
}
type statusType struct {
values map[string]int
mux sync.Mutex
}
var status = statusType{values: make(map[string]int)}
func (s *statusType) set(url string, value int) {
s.mux.Lock()
s.values[url] = value
s.mux.Unlock()
}
func init() {
data, err := ioutil.ReadFile("config.yaml")
if err != nil {
fmt.Printf("Invalid config: %s\n", err)
os.Exit(0)
}
err = yaml.Unmarshal(data, &config)
if err != nil {
fmt.Printf("Invalid config: %s\n", err)
os.Exit(0)
}
for _, url := range config.Urls {
status.set(url, 200)
}
}
func main() {
ticker := time.NewTicker(time.Duration(config.Frequency) * time.Second)
for _ = range ticker.C {
for _, url := range config.Urls {
go check(url)
}
}
}
func check(url string) {
res, err := http.Get(url)
if err != nil {
res = &http.Response{StatusCode: 500}
}
// the memory problem occurs when this condition is never satisfied, so I didn't post the slack package.
if res.StatusCode != status.values[url] {
status.set(url, res.StatusCode)
err := slack.Alert(url, res.StatusCode)
if err != nil {
fmt.Println(err)
}
}
}
If this belongs in Code Review then I will put it there.
Yes, this is a memory leak. One obvious source I can spot is that you're not closing the response bodies from your requests:
func check(url string) {
res, err := http.Get(url)
if err != nil {
res = &http.Response{StatusCode: 500}
} else {
defer res.Body.Close() // You need to close the response body!
}
if res.StatusCode != status.values[url] {
status.set(url, res.StatusCode)
err := slack.Alert(url, res.StatusCode)
if err != nil {
fmt.Println(err)
}
}
}
Better still, so that Go can use keepalive, you want to read the full body and close it:
defer func() {
io.Copy(ioutil.Discard, res.Body)
res.Body.Close()
}()
You can further analyse where memory usage is coming from by profiling your application with pprof. There's a good rundown on the Go blog and a web search will turn up many more articles on the topic.

Resources