go - Simple worker pool - go

As a beginner in go and programming in general, I've been coding a port scanner in go with gopacket library and most of the code is done, but I've been running into a problem of spawning too many goroutines and getting 'read ip4 0.0.0.0: i/o timeout' I've done some research and it seems I need to implement a worker pool I've been trying to implement this example 'https://gobyexample.com/worker-pools' as I'm still learning goroutines and channels I've been at it a few days now and can't seem to figure out how to implement the above example correctly in my program can you guys give me some pointers or preferably a code fix example.
package main
import (
"fmt"
"log"
"net"
"time"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
)
// Get preferred outbound ip and port of this machine
func GetOutboundIPPort() (net.IP, int) {
conn, err := net.Dial("udp", "1.1.1.1:80")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
localAddr := conn.LocalAddr().(*net.UDPAddr)
return localAddr.IP, localAddr.Port
}
func ipv4_gen(out chan net.IP) {
ip, ipnet, err := net.ParseCIDR("192.168.0.0/24")
if err != nil {
log.Fatal(err)
}
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
time.Sleep(300 * time.Millisecond)
dstaddrs, err := net.LookupIP(ip.String())
if err != nil {
log.Fatal(err)
}
dstip := dstaddrs[0].To4()
out <- dstip
}
close(out)
}
func inc(ip net.IP) {
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}
func port_scanner(dstip net.IP) {
dstport := layers.TCPPort(80)
srcip, port := GetOutboundIPPort()
srcport := layers.TCPPort(port)
ip := &layers.IPv4{
SrcIP: srcip,
DstIP: dstip,
Protocol: layers.IPProtocolTCP,
}
tcp := &layers.TCP{
SrcPort: srcport,
DstPort: dstport,
Seq: 1105024978,
SYN: true,
Window: 14600,
}
tcp.SetNetworkLayerForChecksum(ip)
buf := gopacket.NewSerializeBuffer()
opts := gopacket.SerializeOptions{
ComputeChecksums: true,
FixLengths: true,
}
if err := gopacket.SerializeLayers(buf, opts, tcp); err != nil {
log.Fatal(err)
}
conn, err := net.ListenPacket("ip4:tcp", "0.0.0.0")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
if _, err := conn.WriteTo(buf.Bytes(), &net.IPAddr{IP: dstip}); err != nil {
log.Fatal(err)
}
// Set deadline so we don't wait forever.
if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
log.Fatal(err)
}
for {
b := make([]byte, 4096)
n, addr, err := conn.ReadFrom(b)
if err != nil {
log.Println("error reading packet: ", err)
return
} else if addr.String() == dstip.String() {
// Decode a packet
packet := gopacket.NewPacket(b[:n], layers.LayerTypeTCP, gopacket.Default)
// Get the TCP layer from this packet
if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
tcp, _ := tcpLayer.(*layers.TCP)
if tcp.DstPort == srcport {
if tcp.SYN && tcp.ACK {
fmt.Printf("Discovered open port %d/tcp on %s\n", dstport, dstip)
}
return
}
}
}
}
}
func worker(id int, ips <-chan net.IP) {
for ip := range ips {
go port_scanner(ip)
}
}
func main() {
ips := make(chan net.IP)
go ipv4_gen(ips)
for w := 1; w <= 10; w++ {
go worker(w, ips)
}
}

You are starting a new goroutine for each piece of work inside the worker so it defeats its purpose.
You need to run the work instead of starting a goroutine inside the worker.

Related

go udp requests (packet) loss

i wrote simple server and client for tcp and udp connection
package main
//server.go
import (
"fmt"
"net"
"os"
"os/signal"
"syscall"
)
func main() {
tcp := 0
udp := 0
defer func(o, t *int) {
fmt.Println(*o, *t)
}(&tcp, &udp)
go func() {
l, err := net.ListenTCP("tcp", &net.TCPAddr{
IP: net.ParseIP("0.0.0.0"),
Port: 3000,
})
if err != nil {
panic(err)
}
b := make([]byte, 24)
for {
conn, err := l.Accept()
if err != nil {
continue
}
n, err := conn.Read(b)
if err != nil {
continue
}
r := string(b[:n])
if r == "close" {
conn.Close()
break
} else {
_, err = conn.Write([]byte("pong"))
if err != nil {
continue
}
}
conn.Close()
tcp++
}
l.Close()
}()
go func() {
conn, err := net.ListenUDP("udp", &net.UDPAddr{
IP: net.ParseIP("0.0.0.0"),
Port: 3000,
})
if err != nil {
panic(err)
}
b := make([]byte, 24)
for {
n, addr, err := conn.ReadFromUDP(b)
if err != nil {
continue
}
r := string(b[:n])
if r == "close" {
break
} else {
_, err = conn.WriteToUDP([]byte("pong"), addr)
if err != nil {
continue
}
}
udp++
}
conn.Close()
}()
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
<-signals
}
package main
//client.go
import (
"fmt"
"net"
"os"
"os/signal"
"strconv"
"sync/atomic"
"syscall"
"time"
)
func main() {
t := "tcp"
m := "ping"
c := 1
if len(os.Args) > 1 {
t = os.Args[1]
}
if len(os.Args) > 2 {
m = os.Args[2]
}
if len(os.Args) > 3 {
c, _ = strconv.Atoi(os.Args[3])
}
tcp := int64(0)
udp := int64(0)
defer func(o, t *int64) {
fmt.Println(*o, *t)
}(&tcp, &udp)
if c == 1 {
if t == "tcp" {
addr, err := net.ResolveTCPAddr("tcp", ":3000")
if err != nil {
panic(err)
}
conn, err := net.DialTCP("tcp", nil, addr)
if err != nil {
panic(err)
}
_, err = conn.Write([]byte(m))
if err != nil {
panic(err)
}
tcp++
}
if t == "udp" {
addr, err := net.ResolveUDPAddr("udp", ":3000")
if err != nil {
panic(err)
}
conn, err := net.DialUDP("udp", nil, addr)
if err != nil {
panic(err)
}
_, err = conn.Write([]byte(m))
if err != nil {
panic(err)
}
udp++
}
os.Exit(0)
}
for i := 0; i < c; i++ {
go func() {
a1, err := net.ResolveTCPAddr("tcp", ":3000")
if err != nil {
panic(err)
}
c1, err := net.DialTCP("tcp", nil, a1)
if err != nil {
panic(err)
}
_, err = c1.Write([]byte(m))
if err != nil {
panic(err)
}
buf := make([]byte, 24)
n, err := c1.Read(buf)
if err != nil {
panic(err)
}
if string(buf[:n]) != "pong" {
panic(1)
}
err = c1.Close()
if err != nil {
panic(err)
}
g := atomic.AddInt64(&tcp, 1)
if g % 100 == 0 {
fmt.Println("tcp", g)
time.Sleep(time.Millisecond * 1000)
}
}()
go func() {
a2, err := net.ResolveUDPAddr("udp", ":3000")
if err != nil {
panic(err)
}
c2, err := net.DialUDP("udp", nil, a2)
if err != nil {
panic(err)
}
_, err = c2.Write([]byte(m))
if err != nil {
panic(err)
}
buf := make([]byte, 24)
n, err := c2.Read(buf)
if err != nil {
panic(err)
}
if string(buf[:n]) != "pong" {
panic(1)
}
err = c2.Close()
if err != nil {
panic(err)
}
g := atomic.AddInt64(&udp, 1)
if g % 100 == 0 {
fmt.Println("udp", g)
time.Sleep(time.Millisecond * 1000)
}
}()
}
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
<-signals
}
and get a strange behavior: not all udp requests are sent or handled on many connections.
When i sent 100 or 200 both server and client tells me that all requests worked but from 1000 there is a strange ~5% requests loss on udp for both server and client but no panic.
I know udp allows packet loss but 5% on localhost requests seems like an error.
As we know, the UDP is connectionless, so packet loss may be caused by this nature of UDP. There could be several ways to lower the rate of packet loss.
Slow down the rate of packets sent on the client side
Call SetWriteBuffer to increase the buffer size on the client side and set SetReadBuffer to increase the buffer size on the server side
conn, err := net.DialUDP("udp", nil, addr)
err = conn.SetWriteBuffer(64 * 1024 * 1024)
Check the system network information through netstat -s -udp and get UDP statistics information. You could try to change the default value of rmem_max and rmem_default on the server side. For more details, please refer to 1 and 2

Cannot receive response packets from multicast server in golang

I setup a udp server listening multicast trafic, and create a client to sent test packet. When the server receives the package, it will send a response. Everything is ok, except the client cannot receive the packet from the server. Why?
package main
import (
"log"
"net"
"os"
"time"
"golang.org/x/net/ipv4"
)
func SetupMulticast(ifn, addr string) {
ifi, err := net.InterfaceByName(ifn)
if err != nil {
log.Fatal(err)
}
//server
c, err := net.ListenPacket("udp4", addr)
if err != nil {
log.Fatal(err)
}
log.Printf("listen ad:%s\n", addr)
p := ipv4.NewPacketConn(c)
gAddr, err2 := net.ResolveUDPAddr("udp4", addr)
if err2 != nil {
log.Fatal(err2)
}
if err := p.JoinGroup(ifi, gAddr); err != nil {
log.Fatal(err)
}
if err := p.SetControlMessage(ipv4.FlagDst|ipv4.FlagSrc, true); err != nil {
log.Fatal(err)
}
go func() {
b := make([]byte, 1500)
for {
n, cm, src, err := p.ReadFrom(b)
if err != nil {
break
}
log.Println("receive:", string(b[:n]), cm.Dst.IsMulticast(), cm.Dst)
if n2, err := p.WriteTo([]byte("world!"), nil, src); err != nil {
log.Printf("fail to write back:%v\n", err)
} else {
log.Printf("write back addr: %s length:%d\n", src, n2)
}
}
}()
//client
if conn, err2 := net.DialUDP("udp4", nil /*src*/, gAddr); err2 != nil {
log.Fatal(err)
} else {
go func() {
for {
log.Println("write hello...")
conn.Write([]byte("hello"))
time.Sleep(time.Second * 2)
}
}()
go func() {
b2 := make([]byte, 1500)
for {
n, err := conn.Read(b2)
if err != nil {
log.Panic(err)
}
log.Printf("sender received response:%s\n", string(b2[:n]))
}
}()
}
}
func main() {
If := os.Args[1] //ens33
Addr := os.Args[2] //224.0.0.248:1025
SetupMulticast(If, Addr)
for {
}
}
And the output:
2022/08/17 22:53:53 listen ad:224.0.0.248:1025
2022/08/17 22:53:53 write hello...
2022/08/17 22:53:53 receive: hello true 224.0.0.248
2022/08/17 22:53:53 write back addr: 192.168.19.131:43925 length:6
2022/08/17 22:53:55 write hello...
2022/08/17 22:53:55 receive: hello true 224.0.0.248
2022/08/17 22:53:55 write back addr: 192.168.19.131:43925 length:6
From the logs, there are no any "sender received response" record. I don't know why?

How to find server real IP

How do I find the public IP address of the machine or server my program is running on?
Like when the program is executed it detect the public of the server and print for e.g.running at 123.45.67.89
The short answer is that there's no method guaranteed to return your "public" ip address.
The first question is, what is your public ip address? The address of your machine, as seen by the system to which you are connecting, may vary depending on how your local Internet service is configured and on the service to which you're connecting:
As I mentioned in a comment, in a typical home environment your machine doesn't have a public ip address. The public address is hosted by your router.
If you're accessing a service through a proxy or vpn, your machine's address may be entirely different from when you're directly connecting to a service.
On a system with multiple interfaces, the origin address selected may depend upon the address to which you are connecting: different addresses may have different routes.
You can try using a service like http://icanhazip.com/ to try to determine your public ip. This will be correct in many but not all situations.
public IP address is a vague notion, in practice, it might or might not be a static address. What do you know about that ? It is just a endpoint valid for a certain amount of time, which depends on many factors like which interface was used to issue the query.
We can use the mainline bittorrent dht to give us some indications.
The Go language provides the cool dht package written by anacrolix.
When querying nodes with a find_peers verb we receive a packet containing the remote ip address the peer has associated with our query. This is described in bep10.
If an UDP connection is not a good option, you might opt for a query to bittorent trackers as described in bep24
Consider that peers might be malicious, thus the more results, the better.
Below program outputs the list of external network addresses associated with the computer initiating the query from the POV of the cohort of nodes queried.
Addresses are scored by the numbers of response.
read also https://www.bittorrent.org/beps/bep_0005.html
found 9 bootstrap peers
found 6 peers
4 [2001:861:51c5:xxx:40d1:8061:1fe0:xxx]:9090
2 81.96.42.191:9090
4 peers told us that we were using [2001:861:51c5:xxx:40d1:8061:1fe0:xxx]:9090, we can infer this is ipv6.
2 of them told we were using 81.96.42.191:9090, the ipv4 interface.
package main
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net"
"os"
"sort"
"sync"
"time"
"github.com/anacrolix/dht"
"github.com/anacrolix/dht/krpc"
"github.com/anacrolix/torrent/bencode"
)
var maxTimeout = time.Second * 5
func main() {
b, _ := ioutil.ReadFile("db.json")
var rawAddrs []string
json.Unmarshal(b, &rawAddrs)
defer func() {
if len(rawAddrs) < 1 {
return
}
if len(rawAddrs) > 30 {
rawAddrs = rawAddrs[:30]
}
buf, err := json.Marshal(rawAddrs)
if err != nil {
panic(err)
}
err = ioutil.WriteFile("db.json", buf, os.ModePerm)
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "%v peers recorded\n", len(rawAddrs))
}()
bootstrap, err := parseAddrs(rawAddrs)
if err != nil {
bootstrap, err = globalBootstrapAddrs()
if err != nil {
panic(err)
}
}
findPeers := []byte(`d1:ad2:id20:abcdefghij01234567899:info_hash20:mnopqrstuvwxyz123456e1:q9:get_peers1:t2:aa1:y1:qe`)
local, err := net.ResolveUDPAddr("udp", "0.0.0.0:9090")
if err != nil {
panic(err)
}
ln, err := net.ListenUDP("udp", local)
if err != nil {
panic(err)
}
addrscores := map[string]int{}
var drain drain
defer drain.Wait()
fmt.Fprintf(os.Stderr, "found %v bootstrap peers\n", len(bootstrap))
res, errs := readResponses(ln, len(bootstrap), sendQuery(ln, bootstrap, findPeers))
drain.Errors(errs)
peers := []net.Addr{}
for d := range res {
if isValidAddr(d.IP.UDP()) {
addrscores[d.IP.String()]++
d.R.ForAllNodes(func(arg1 krpc.NodeInfo) {
peers = append(peers, arg1.Addr.UDP())
})
}
}
if len(peers) > 0 {
fmt.Fprintf(os.Stderr, "found %v peers\n", len(peers))
res, errs = readResponses(ln, len(peers), sendQuery(ln, peers, findPeers))
drain.Errors(errs)
for d := range res {
if isValidAddr(d.IP.UDP()) {
addrscores[d.IP.String()]++
}
}
}
for _, peer := range peers {
if isValidAddr(peer) {
rawAddrs = append(rawAddrs, peer.String())
}
}
addrs := make([]string, 0, len(addrscores))
for addr := range addrscores {
addrs = append(addrs, addr)
}
sort.Slice(addrs, func(i int, j int) bool {
return addrscores[addrs[i]] > addrscores[addrs[j]]
})
for _, addr := range addrs {
fmt.Printf("%-4v %v\n", addrscores[addr], addr)
}
}
type drain struct{ sync.WaitGroup }
func (d *drain) Errors(errs <-chan error) {
d.Add(1)
go func() {
defer d.Done()
for err := range errs {
fmt.Fprintln(os.Stderr, err)
}
}()
}
func parseAddrs(rawAddrs []string) (addrs []net.Addr, err error) {
for _, s := range rawAddrs {
host, port, err := net.SplitHostPort(s)
if err != nil {
panic(err)
}
ua, err := net.ResolveUDPAddr("udp", net.JoinHostPort(host, port))
if err != nil {
log.Printf("error resolving %q: %v", host, err)
continue
}
addrs = append(addrs, ua)
}
if len(addrs) == 0 {
err = errors.New("nothing resolved")
}
return
}
func globalBootstrapAddrs() (addrs []net.Addr, err error) {
bootstrap, err := dht.GlobalBootstrapAddrs("udp")
if err != nil {
return nil, err
}
for _, b := range bootstrap {
addrs = append(addrs, b.Raw())
}
return
}
func isValidAddr(addr net.Addr) bool { // so weird guys.
return addr.String() != "<nil>" && addr.String() != ":0"
}
func sendQuery(ln *net.UDPConn, peers []net.Addr, query []byte) chan error {
errs := make(chan error)
for _, addr := range peers {
go func(addr net.Addr) {
_, err := ln.WriteTo(query, addr)
if err != nil {
errs <- addressedError{Op: "send", error: err, Addr: addr}
}
}(addr)
}
return errs
}
func readResponses(ln *net.UDPConn, count int, errs chan error) (<-chan krpc.Msg, <-chan error) {
data := make(chan krpc.Msg)
var wg sync.WaitGroup
for i := 0; i < count; i++ {
wg.Add(1)
go func() {
defer wg.Done()
buf := make([]byte, 1000)
ln.SetReadDeadline(time.Now().Add(maxTimeout))
n, remoteAddr, err := ln.ReadFromUDP(buf)
if err != nil {
errs <- addressedError{Op: "rcv", error: err, Addr: remoteAddr}
return
}
var m krpc.Msg
err = bencode.Unmarshal(buf[:n], &m)
if err != nil {
errs <- addressedError{Op: "rcv", error: err, Addr: remoteAddr}
return
}
data <- m
}()
}
go func() {
wg.Wait()
close(errs)
close(data)
}()
return data, errs
}
type addressedError struct {
error
Op string
Addr net.Addr
}
func (a addressedError) Error() string {
if !isValidAddr(a.Addr) {
return fmt.Sprintf("%-5v %v", a.Op, a.error.Error())
}
return fmt.Sprintf("%-5v %v: %v", a.Op, a.Addr.String(), a.error.Error())
}

Golang TCPConn Gob Communication

I'm having issues with the gob protocol (or maybe networking in general, where my knowledge is weak), and I don't understand why the following code does not work properly. It is just supposed to be a simple example of maintaining an open TCP connection, and sending multiple gobs through it. The code will send, and receive, but often corrupts its data.
Thank you in advance.
package main
import (
"encoding/gob"
"fmt"
"net"
"strconv"
"time"
)
type Message struct {
Msg string
}
func main() {
gob.Register(new(Message))
clientAddr, err := net.ResolveTCPAddr("tcp", "localhost:12346")
if err != nil {
fmt.Println(err)
}
serverAddr, err := net.ResolveTCPAddr("tcp", "localhost:12345")
if err != nil {
fmt.Println(err)
}
serverListener, err := net.ListenTCP("tcp", serverAddr)
if err != nil {
fmt.Println(err)
}
conn, err := net.DialTCP("tcp", clientAddr, serverAddr)
if err != nil {
fmt.Println(err)
}
serverConn, err := serverListener.AcceptTCP()
if err != nil {
fmt.Println(err)
}
done := false
go func() {
for !done {
recieveMessage(serverConn)
}
}()
for i := 1; i < 1000; i++ {
sent := Message{strconv.Itoa(i)}
sendMessage(sent, conn)
}
time.Sleep(time.Second)
done = true
}
func sendMessage(msg Message, conn *net.TCPConn) {
enc := gob.NewEncoder(conn)
err := enc.Encode(msg)
if err != nil {
fmt.Println(err)
}
}
func recieveMessage(conn *net.TCPConn) {
msg := new(Message)
dec := gob.NewDecoder(conn) // Will read from network.
err := dec.Decode(msg)
if err != nil {
fmt.Println(err)
}
fmt.Println("Client recieved:", msg.Msg)
}
The problem is that the decoder can buffer data from the next message. When this happens, the next new decoder starts in the middle of a message. The fix is to use a single encoder and decoder.
func main() {
...
dec := gob.NewDecoder(conn) // Will read from network.
enc := gob.NewEncoder(serverConn)
go func() {
for !done {
recieveMessage(dec)
}
}()
for i := 1; i < 1000; i++ {
sent := Message{strconv.Itoa(i)}
sendMessage(sent, enc)
}
...
}
func sendMessage(msg Message, enc *gob.Encoder) {
err := enc.Encode(msg)
if err != nil {
fmt.Println(err)
}
}
func recieveMessage(dec *gob.Decoder) {
msg := new(Message)
err := dec.Decode(msg)
if err != nil {
fmt.Println(err)
}
fmt.Println("Client recieved:", msg.Msg)
}
Run it in the playground

Memory management in Go

I have a closure where I declare and define local variables:
func writer_factory() func() *net.TCPConn {
response_port := "localhost:8000"
tcpAddr_res, err := net.ResolveTCPAddr("tcp4", response_port)
checkError(err)
var response_writer *net.TCPConn
checkError(err)
return func() *net.TCPConn {
if response_writer == nil{
response_writer, err = net.DialTCP("tcp", nil, tcpAddr_res)
checkError(err)
}
return response_writer
}
}
Now my question is, if I call this writer_factory multiple times will I get a memory leak?
So specifically, will I have a memory leak using writer_factory in this program:
package main
import (
"fmt"
"net"
"os"
"strings"
// "io/ioutil"
)
//fmt.Printf("messages are (1) %q\n", messages)
func main() {
end_of_message_terminator := "||"
beginning_of_next_message := ""
request := make([]byte, 512)
service_port := ":7777"
tcpAddr, err := net.ResolveTCPAddr("tcp4", service_port)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
for {
response_writer := writer_factory()
conn, err := listener.Accept()
if err != nil {
continue
}
read_len, err := conn.Read(request)
if read_len == 0 {
continue
}
request_string := string(request[:read_len])
messages := strings.Split(request_string, end_of_message_terminator)
messages[0] = beginning_of_next_message + messages[0]
if messages[len(messages) - 1] != "" {
beginning_of_next_message = messages[len(messages) - 1]
messages[len(messages) - 1] = ""
}
if len(messages) == 1 {
continue
}
rw := response_writer()
join_channel := make(chan struct{})
for i := 0; i < len(messages); i++ {
go func(i int, rw *net.TCPConn){
respond_to_message(messages[i], rw)
join_channel <- struct{}{}
}(i, rw)
}
go func (){
for i := 0; i < len(messages); i++ {
<- join_channel
}
rw.Close()
}()
conn.Close()
}
}
func writer_factory() func() *net.TCPConn {
response_port := "localhost:8000"
tcpAddr_res, err := net.ResolveTCPAddr("tcp4", response_port)
checkError(err)
var response_writer *net.TCPConn
checkError(err)
return func() *net.TCPConn {
if response_writer == nil{
response_writer, err = net.DialTCP("tcp", nil, tcpAddr_res)
checkError(err)
}
return response_writer
}
}
func respond_to_message(message string, response_writer *net.TCPConn){
message_parameters := strings.Split(message, "|")
//response_writer.Write([]byte("asti de chris"))
for i := range message_parameters {
param_parts := strings.Split(message_parameters[i], "=")
param_name := param_parts[0]
//param_value := param_parts[1]
response_writer.Write([]byte(param_name))
//fmt.Println(string(result))
}
}
func checkError(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "Fatal error: %s", err.Error())
os.Exit(1)
}
}
To exercise this code launch the program, then launch this program netcat -l -p 8000 and then this program printf "asti||" | netcat localhost 7777
You can definitely improve performance by using one connection or (if one connection is not thread save) use a sync.Pool. That would save you memory allocations (and with it garbage collection) and also time wasted on opening all those tcp connections.

Resources