Proper way for cancelling accept(block) and close listener in Go [duplicate] - go

I've been trying to find a way to stop a listening server in Go gracefully. Because listen.Accept blocks it is necessary to close the listening socket to signal the end, but I can't tell that error apart from any other errors as the relevant error isn't exported.
Can I do better than this? See FIXME in the code below in serve()
package main
import (
"io"
"log"
"net"
"time"
)
// Echo server struct
type EchoServer struct {
listen net.Listener
done chan bool
}
// Respond to incoming connection
//
// Write the address connected to then echo
func (es *EchoServer) respond(remote *net.TCPConn) {
defer remote.Close()
_, err := io.Copy(remote, remote)
if err != nil {
log.Printf("Error: %s", err)
}
}
// Listen for incoming connections
func (es *EchoServer) serve() {
for {
conn, err := es.listen.Accept()
// FIXME I'd like to detect "use of closed network connection" here
// FIXME but it isn't exported from net
if err != nil {
log.Printf("Accept failed: %v", err)
break
}
go es.respond(conn.(*net.TCPConn))
}
es.done <- true
}
// Stop the server by closing the listening listen
func (es *EchoServer) stop() {
es.listen.Close()
<-es.done
}
// Make a new echo server
func NewEchoServer(address string) *EchoServer {
listen, err := net.Listen("tcp", address)
if err != nil {
log.Fatalf("Failed to open listening socket: %s", err)
}
es := &EchoServer{
listen: listen,
done: make(chan bool),
}
go es.serve()
return es
}
// Main
func main() {
log.Println("Starting echo server")
es := NewEchoServer("127.0.0.1:18081")
// Run the server for 1 second
time.Sleep(1 * time.Second)
// Close the server
log.Println("Stopping echo server")
es.stop()
}
This prints
2012/11/16 12:53:35 Starting echo server
2012/11/16 12:53:36 Stopping echo server
2012/11/16 12:53:36 Accept failed: accept tcp 127.0.0.1:18081: use of closed network connection
I'd like to hide the Accept failed message, but obviously I don't want to mask other errors Accept can report. I could of course look in the error test for use of closed network connection but that would be really ugly. I could set a flag saying I'm about to close and ignore errors if that was set I suppose - Is there a better way?

I would handle this by using es.done to send a signal before it closes the connection. In addition to the following code you'd need to create es.done with make(chan bool, 1) so that we can put a single value in it without blocking.
// Listen for incoming connections
func (es *EchoServer) serve() {
for {
conn, err := es.listen.Accept()
if err != nil {
select {
case <-es.done:
// If we called stop() then there will be a value in es.done, so
// we'll get here and we can exit without showing the error.
default:
log.Printf("Accept failed: %v", err)
}
return
}
go es.respond(conn.(*net.TCPConn))
}
}
// Stop the server by closing the listening listen
func (es *EchoServer) stop() {
es.done <- true // We can advance past this because we gave it buffer of 1
es.listen.Close() // Now it the Accept will have an error above
}

Check some "is it time to stop" flag in your loop right after the accept() call, then flip it from your main, then connect to your listening port to get server socket "un-stuck". This is very similar to the old "self-pipe trick".

Something among these lines might work in this case, I hope:
// Listen for incoming connections
func (es *EchoServer) serve() {
for {
conn, err := es.listen.Accept()
if err != nil {
if x, ok := err.(*net.OpError); ok && x.Op == "accept" { // We're done
log.Print("Stoping")
break
}
log.Printf("Accept failed: %v", err)
continue
}
go es.respond(conn.(*net.TCPConn))
}
es.done <- true
}

Here's a simple way that's good enough for local development.
http://www.sergiotapia.me/how-to-stop-your-go-http-server/
package main
import (
"net/http"
"os"
"github.com/bmizerany/pat"
)
var mux = pat.New()
func main() {
mux.Get("/kill", http.HandlerFunc(kill))
http.Handle("/", mux)
http.ListenAndServe(":8080", nil)
}
func kill(w http.ResponseWriter, r *http.Request) {
os.Exit(0)
}

Related

Cancelling a net.Listener via Context in Golang

I'm implementing a TCP server application that accepts incoming TCP connections in an infinite loop.
I'm trying to use Context throughout the application to allow shutting down, which is generally working great.
The one thing I'm struggling with is cancelling a net.Listener that is waiting on Accept(). I'm using a ListenConfig which, I believe, has the advantage of taking a Context when then creating a Listener. However, cancelling this Context does not have the intended effect of aborting the Accept call.
Here's a small app that demonstrates the same problem:
package main
import (
"context"
"fmt"
"net"
"time"
)
func main() {
lc := net.ListenConfig{}
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(2*time.Second)
fmt.Println("cancelling context...")
cancel()
}()
ln, err := lc.Listen(ctx, "tcp", ":9801")
if err != nil {
fmt.Println("error creating listener:", err)
} else {
fmt.Println("listen returned without error")
defer ln.Close()
}
conn, err := ln.Accept()
if err != nil {
fmt.Println("accept returned error:", err)
} else {
fmt.Println("accept returned without error")
defer conn.Close()
}
}
I expect that, if no clients connect, when the Context is cancelled 2 seconds after startup, the Accept() should abort. However, it just sits there until you Ctrl-C out.
Is my expectation wrong? If so, what is the point of the Context passed to ListenConfig.Listen()?
Is there another way to achieve the same goal?
I believe you should be closing the listener when your timeout runs out. Then, when Accept returns an error, check that it's intentional (e.g. the timeout elapsed).
This blog post shows how to do a safe shutdown of a TCP server without a context. The interesting part of the code is:
type Server struct {
listener net.Listener
quit chan interface{}
wg sync.WaitGroup
}
func NewServer(addr string) *Server {
s := &Server{
quit: make(chan interface{}),
}
l, err := net.Listen("tcp", addr)
if err != nil {
log.Fatal(err)
}
s.listener = l
s.wg.Add(1)
go s.serve()
return s
}
func (s *Server) Stop() {
close(s.quit)
s.listener.Close()
s.wg.Wait()
}
func (s *Server) serve() {
defer s.wg.Done()
for {
conn, err := s.listener.Accept()
if err != nil {
select {
case <-s.quit:
return
default:
log.Println("accept error", err)
}
} else {
s.wg.Add(1)
go func() {
s.handleConection(conn)
s.wg.Done()
}()
}
}
}
func (s *Server) handleConection(conn net.Conn) {
defer conn.Close()
buf := make([]byte, 2048)
for {
n, err := conn.Read(buf)
if err != nil && err != io.EOF {
log.Println("read error", err)
return
}
if n == 0 {
return
}
log.Printf("received from %v: %s", conn.RemoteAddr(), string(buf[:n]))
}
}
In your case you should call Stop when the context runs out.
If you look at the source code of TCPConn.Accept, you'll see it basically calls the underlying socket accept, and the context is not piped through there. But Accept is simple to cancel by closing the listener, so piping the context all the way isn't strictly necessary.

How to set SetKeepAlivePeriod on a *tls.Conn

I want to increase the keep alive period of my TCP connection for both HTTP and HTTPS requests.
For HTTP requests this can be done like this:
package main
import (
"fmt"
"io"
"log"
"net"
"net/http"
"time"
)
func main() {
server := &http.Server{Addr: ":8080", Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "Hello, World!")
})}
server.ConnState = func(conn net.Conn, state http.ConnState) {
if state == http.StateNew {
if err := conn.(*net.TCPConn).SetKeepAlivePeriod(1000 * time.Second); err != nil {
fmt.Println("Could not set keep alive period", err)
} else {
fmt.Println("update keep alive period")
}
}
}
log.Fatal(server.ListenAndServe())
}
For HTTPS requests this cannot be done via server.ConnState because the net.Conn that will be passed inside the function is a *tls.Conn. This connection does not expose a function like SetKeepAlivePeriod or gives access to the underlying *net.TCPConn.
func main() {
server := &http.Server{Addr: ":8080", Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "Hello, World!")
})}
server.ConnState = func(conn net.Conn, state http.ConnState) {
if state == http.StateNew {
tlsConn := conn.(*tls.Conn)
// how to set SetKeepAlivePeriod
}
}
log.Fatal(server.ListenAndServeTLS("../example.crt", "../example.key"))
}
How can I set the keep alive period for tls connections?
There are (at least) two ways to do it:
Use a net.ListenConfig:
The net.ListenConfig object has a KeepAlive time.Duration field. When non-zero, this will be used to set the keep-alive on accepted connections (eg: for TCP on posix).
You can pass the listener to ServeTLS:
server := &http.Server{...}
lc := net.ListenConfig{KeepAlive: 1000 * time.Second}
ln, err := lc.Listen(context.Background(), "tcp", ":8080")
if err != nil {
panic(err)
}
defer ln.Close()
log.Fatal(server.ServeTLS(ln, "../example.crt", "../example.key"))
As mentioned, accepted TCP connections will automatically have keep-alive enabled and the period set to the specified value.
Use a tls.Config callback:
You can access the net.Conn underlying the tls.Conn by setting the tls.Config GetConfigForClient or GetCertificate callback.
It does not matter which one you're using as long as you return nil to make the TLS code fall back to the default behavior. The important part is to get access to the tls.ClientHelloInfo which has a .Conn field pointing to the underlying connection. This will the net.TCPConn.
setTCPKeepAlive := func(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
// Check that the underlying connection really is TCP.
if tcpConn, ok := clientHello.Conn.(*net.TCPConn); ok {
if err := tcpConn.SetKeepAlivePeriod(1000 * time.Second); err != nil {
fmt.Println("Could not set keep alive period", err)
} else {
fmt.Println("update keep alive period")
}
} else {
fmt.Println("TLS over non-TCP connection")
}
// Make sure to return nil, nil to let the caller fall back on the default behavior.
return nil, nil
}
tlsConfig := &tls.Config{
...
GetConfigForClient: setTCPKeepAlive,
...
}
server := &http.Server{
Addr: ":8080",
TLSConfig: tlsConfig,
}
server.ListenAndServeTLS("../example.crt", "../example.key")

How to stop http.ListenAndServe()

I am using the Mux library from Gorilla Web Toolkit along with the bundled Go http server.
The problem is that in my application the HTTP server is only one component and it is required to stop and start at my discretion.
When I call http.ListenAndServe(fmt.Sprintf(":%d", service.Port()), service.router) it blocks and I cannot seem to stop the server from running.
I am aware this has been a problem in the past, is that still the case? Are there any new solutions?
Regarding graceful shutdown (introduced in Go 1.8), a bit more concrete example:
package main
import (
"context"
"io"
"log"
"net/http"
"sync"
"time"
)
func startHttpServer(wg *sync.WaitGroup) *http.Server {
srv := &http.Server{Addr: ":8080"}
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "hello world\n")
})
go func() {
defer wg.Done() // let main know we are done cleaning up
// always returns error. ErrServerClosed on graceful close
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
// unexpected error. port in use?
log.Fatalf("ListenAndServe(): %v", err)
}
}()
// returning reference so caller can call Shutdown()
return srv
}
func main() {
log.Printf("main: starting HTTP server")
httpServerExitDone := &sync.WaitGroup{}
httpServerExitDone.Add(1)
srv := startHttpServer(httpServerExitDone)
log.Printf("main: serving for 10 seconds")
time.Sleep(10 * time.Second)
log.Printf("main: stopping HTTP server")
// now close the server gracefully ("shutdown")
// timeout could be given with a proper context
// (in real world you shouldn't use TODO()).
if err := srv.Shutdown(context.TODO()); err != nil {
panic(err) // failure/timeout shutting down the server gracefully
}
// wait for goroutine started in startHttpServer() to stop
httpServerExitDone.Wait()
log.Printf("main: done. exiting")
}
As mentioned in yo.ian.g's answer. Go 1.8 has included this functionality in the standard lib.
Minimal example for for Go 1.8+:
server := &http.Server{Addr: ":8080", Handler: handler}
go func() {
if err := server.ListenAndServe(); err != nil {
// handle err
}
}()
// Setting up signal capturing
stop := make(chan os.Signal, 1)
signal.Notify(stop, os.Interrupt)
// Waiting for SIGINT (kill -2)
<-stop
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := server.Shutdown(ctx); err != nil {
// handle err
}
// Wait for ListenAndServe goroutine to close.
You can kill the server gracefully using kill -2 <pid>
Original Answer - Pre Go 1.8 :
Building on Uvelichitel's answer.
You can create your own version of ListenAndServe which returns an io.Closer and does not block.
func ListenAndServeWithClose(addr string, handler http.Handler) (io.Closer,error) {
var (
listener net.Listener
srvCloser io.Closer
err error
)
srv := &http.Server{Addr: addr, Handler: handler}
if addr == "" {
addr = ":http"
}
listener, err = net.Listen("tcp", addr)
if err != nil {
return nil, err
}
go func() {
err := srv.Serve(tcpKeepAliveListener{listener.(*net.TCPListener)})
if err != nil {
log.Println("HTTP Server Error - ", err)
}
}()
srvCloser = listener
return srvCloser, nil
}
Full code available here.
The HTTP Server will close with the error
accept tcp [::]:8080: use of closed network connection
Go 1.8 will include graceful and forceful shutdown, available via Server::Shutdown(context.Context) and Server::Close() respectively.
go func() {
httpError := srv.ListenAndServe(address, handler)
if httpError != nil {
log.Println("While serving HTTP: ", httpError)
}
}()
srv.Shutdown(context)
The relevant commit can be found here
You can construct net.Listener
l, err := net.Listen("tcp", fmt.Sprintf(":%d", service.Port()))
if err != nil {
log.Fatal(err)
}
which you can Close()
go func(){
//...
l.Close()
}()
and http.Serve() on it
http.Serve(l, service.router)
Since none of the previous answers say why you can't do it if you use http.ListenAndServe(), I went into the v1.8 http source code and here is what it says:
func ListenAndServe(addr string, handler Handler) error {
server := &Server{Addr: addr, Handler: handler}
return server.ListenAndServe()
}
As you can see the http.ListenAndServe function does not return the server variable. This means you cannot get to 'server' to use the Shutdown command. Therefore, you need to create your own 'server' instance instead of using this function for the graceful shutdown to be implemented.
You can close the server by closing its context.
type ServeReqs func(ctx context.Context, cfg Config, deps ReqHandlersDependencies) error
var ServeReqsImpl = func(ctx context.Context, cfg Config, deps ReqHandlersDependencies) error {
http.Handle(pingRoute, decorateHttpRes(pingHandlerImpl(deps.pingRouteResponseMessage), addJsonHeader()))
server := &http.Server{Addr: fmt.Sprintf(":%d", cfg.port), Handler: nil}
go func() {
<-ctx.Done()
fmt.Println("Shutting down the HTTP server...")
server.Shutdown(ctx)
}()
err := server.ListenAndServeTLS(
cfg.certificatePemFilePath,
cfg.certificatePemPrivKeyFilePath,
)
// Shutting down the server is not something bad ffs Go...
if err == http.ErrServerClosed {
return nil
}
return err
}
And whenever you are ready to close it, call:
ctx, closeServer := context.WithCancel(context.Background())
err := ServeReqs(ctx, etc)
closeServer()
It is possible to solve this using a context.Context using a net.ListenConfig. In my case, I didn't want to use a sync.WaitGroup or http.Server's Shutdown() call, and instead rely on a context.Context (which was closed with a signal).
import (
"context"
"http"
"net"
"net/http/pprof"
)
func myListen(ctx context.Context, cancel context.CancelFunc) error {
lc := net.ListenConfig{}
ln, err := lc.Listen(ctx, "tcp4", "127.0.0.1:6060")
if err != nil {
// wrap the err or log why the listen failed
return err
}
mux := http.NewServeMux()
mux.Handle("/debug/pprof/", pprof.Index)
mux.Handle("/debug/pprof/cmdline", pprof.CmdLine)
mux.Handle("/debug/pprof/profile", pprof.Profile)
mux.Handle("/debug/pprof/symbol", pprof.Symbol)
mux.Handle("/debug/pprof/trace", pprof.Trace)
go func() {
if err := http.Serve(l, mux); err != nil {
cancel()
// log why we shut down the context
return err
}
}()
// If you want something semi-synchronous, sleep here for a fraction of a second
return nil
}
Reproducible example when you do not want your main server to be run in a separate goroutine:
main.go:
package main
import (
"context"
"log"
"net/http"
"os"
"os/signal"
"sync"
"time"
)
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
// wait for 10 seconds before sending OK
time.Sleep(10 * time.Second)
_, _ = w.Write([]byte("OK\n"))
})
server := &http.Server{Addr: ":3333", Handler: nil}
// Creating a waiting group that waits until the graceful shutdown procedure is done
var wg sync.WaitGroup
wg.Add(1)
// This goroutine is running in parallels to the main one
go func() {
// creating a channel to listen for signals, like SIGINT
stop := make(chan os.Signal, 1)
// subscribing to interruption signals
signal.Notify(stop, os.Interrupt)
// this blocks until the signal is received
<-stop
// initiating the shutdown
err := server.Shutdown(context.Background())
// can't do much here except for logging any errors
if err != nil {
log.Printf("error during shutdown: %v\n", err)
}
// notifying the main goroutine that we are done
wg.Done()
}()
log.Println("listening on port 3333...")
err := server.ListenAndServe()
if err == http.ErrServerClosed { // graceful shutdown
log.Println("commencing server shutdown...")
wg.Wait()
log.Println("server was gracefully shut down.")
} else if err != nil {
log.Printf("server error: %v\n", err)
}
}
Open two terminals. In the first run the app, in the second one run curl localhost:3333, then quickly switch to the first one and try to stop the app with CTRL+C
The output should be:
2021/03/12 13:39:49 listening on port 3333...
2021/03/12 13:39:50 user initiated a request
2021/03/12 13:39:54 commencing server shutdown...
2021/03/12 13:40:00 user request is fulfilled
2021/03/12 13:40:01 server was gracefully shut down.
There exists a module which implements (graceful) stopping of Go HTTP servers:
https://github.com/pseidemann/finish
This removes the need of the boilerplate presented in the other answers.
What I've done for such cases where the application is just the server and performing no other function is install an http.HandleFunc for a pattern like /shutdown. Something like
http.HandleFunc("/shutdown", func(w http.ResponseWriter, r *http.Request) {
if <credentials check passes> {
// - Turn on mechanism to reject incoming requests.
// - Block until "in-flight" requests complete.
// - Release resources, both internal and external.
// - Perform all other cleanup procedures thought necessary
// for this to be called a "graceful shutdown".
fmt.Fprint(w, "Goodbye!\n")
os.Exit(0)
}
})
It does not require 1.8. But if 1.8 is available, then that solution can be embedded here instead of the os.Exit(0) call if desirable, I believe.
The code to perform all of that cleanup work is left as an exercise for the reader.
Extra credit if you can say where that cleanup code might be most reasonably be placed, for I would not recommend doing it here, and how this endpoint hit should cause the invocation that code.
More extra credit if you can say where that os.exit(0) call (or whatever process exit you choose to use), given here for illustrative purposes only, would be most reasonably placed.
Yet even more extra credit if you can explain why this mechanism of HTTP server process signaling should be considered above all other such mechanisms thought workable in this case.

Golang: Mixing Gin with an UDP server

I'm trying to use both a UDP server to listen continuously to datagrams and a http server, but the string "UDP server up and listening on port..." and command "server.Run()" are never reached.
package main
import (
"fmt"
"github.com/gin-gonic/gin"
"log"
"net"
)
func handleUDPConnection(conn *net.UDPConn) {
buffer := make([]byte, 8096)
n, addr, err := conn.ReadFromUDP(buffer)
if err != nil {
log.Fatal(err)
} else {
fmt.Println("UDP client: ", addr)
fmt.Println("Received from UDP client: ", string(buffer[:n]))
}
}
func main() {
server := gin.Default()
host, port := "localhost", "41234"
udpAddr, err := net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:%s", host, port))
if err != nil {
log.Fatal(err)
}
conn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
log.Fatal(err)
}
defer conn.Close()
server.GET("/ping", func(c *gin.Context) {
c.JSON(200, gin.H{"message": "pong"})
})
for {
handleUDPConnection(conn)
}
fmt.Sprintf("UDP server up and listening on port %s\n", port)
server.Run()
}
How can I make it work?
There is an infinite loop in your code.
for {
handleUDPConnection(conn)
}
This will repetedly call the handleUDPConnection function until the program exits without ever moving on to
fmt.Sprintf("UDP server up and listening on port %s\n", port)
server.Run()
Perhaps you want to deal with the connections in a go thread. This would be something more like this:
//define an exit variable
keepListening := true
//spawn a go routine (starts the function on another thread*)
go func() {
for keepListening {
handleUDPConnection(conn)
}
}()
//notify the user that the server is listening
fmt.Sprintf("UDP server up and listening on port %s\n", port)
//run the server (I assume this function call is blocking
server.Run()
//stop the go routine when the server is done running
keepListening = false
Hope this helps!
*a goroutine is not a thread. It can be useful/simple to think of it like that, but they are distinctly different. Here's an article explaining some of the differences and advantages.

How to reuse listener/connection ? Golang

I'm trying to connect a computer behind NAT with the internet through a 3rd party server(aka reverse connection). I'm listening on two ports. On one port (dstNet) is connecting the machine behind NAT and on the other port are connecting the internet clients.
The issue is that I don't know how to handle the disconnection of the machine behind NAT. Even if the machine is connecting again the the traffic is not handled sent/written anymore... I get [DEBUG] socks: Copied 0 bytes to client which is my warning of course. Below is the code. It's quite long but I can't find what to trim.
// Make a bridge between dstNet which is
// usually behind NAT and srcNet which is usually a client
// which wants to route the traffic though the NAT machine.
package main
import (
"bufio"
"errors"
log "github.com/golang/glog"
"io"
"net"
"time"
)
const (
// listen on the dstNet so that we can
// create a connection with the NAT client
dstNet = "0.0.0.0:9000"
// listen on srcNet so that we can get traffic
// to forward to dstNet
srcNet = "0.0.0.0:9001"
)
var errCh = make(chan error, 1)
// make a channel to send the reverse connections
var lrCh = make(chan net.Conn, 1)
func listenDst() {
// Listen on the dstNet
lr, err := net.Listen("tcp", dstNet)
if err != nil {
log.Error(err)
errCh <- err
return
}
// accept the connection
for {
lrConn, err := lr.Accept()
if err != nil {
log.Error(err)
errCh <- err
return
}
log.Errorf("sent connection")
// lrConn.SetReadDeadline(time.Now().Add(10 * time.Second))
lrCh <- lrConn
}
}
func main() {
go func() {
for err := range errCh {
if err != nil {
panic(err)
}
}
}()
// listen for the nat server
go listenDst()
// listen for clients to connect
l, err := net.Listen("tcp", srcNet)
if err != nil {
log.Error(err)
panic(err)
}
// accept the connection
for {
conn, err := l.Accept()
if err != nil {
log.Error(err)
panic(err)
}
// serve the connection
go func(conn net.Conn) {
defer conn.Close()
bufConn := bufio.NewReader(conn)
dst := <-lrCh
defer dst.Close()
// Start proxying
errCh2 := make(chan error, 2)
go proxy("target", dst, bufConn, errCh2)
go proxy("client", conn, dst, errCh2)
// Wait
var ei int
for err = range errCh2 {
switch {
case err != nil && err.Error() == "no byte":
log.Error(err)
case err != nil && err.Error() == "use of closed network connection":
// if the connection is closed we restart it.
log.Error(err)
// BUG() attempt to write again the bytes
case err != nil:
log.Error(err)
errCh <- err
}
if ei == 1 {
log.Errorf("done with errors")
close(errCh2)
}
ei++
}
}(conn)
}
}
// proxy is used to suffle data from src to destination, and sends errors
// down a dedicated channel
func proxy(name string, dst io.Writer, src io.Reader, errCh2 chan error) {
n, err := io.Copy(dst, src)
// Log, and sleep. This is jank but allows the otherside
// to finish a pending copy
log.Errorf("[DEBUG] socks: Copied %d bytes to %s", n, name)
time.Sleep(10 * time.Millisecond)
// Send any errors
switch {
case err != nil:
log.Error(err)
errCh2 <- err
case n < 1:
errCh2 <- errors.New("no byte")
default:
errCh2 <- nil
}
return
}
The only time you can reuse a connection after an error is if is a temporary condition.
if err, ok := err.(net.Error); ok && err.Temporary() {
}
If you are trying to proxy a TCPconnection, and there is any other error (checking for Temporary may not even be that useful), you need to drop the whole thing and start over. You have no idea what the state of the remote server is, how many packets are in flight or lost, and it's only going to cause more difficult bugs the harder you try. (tip: don't hide concurrency or timing problems with a sleep. It's just making it harder in the long run)
Here is a much simpler proxy pattern for go if you want to reference it:
https://gist.github.com/jbardin/821d08cb64c01c84b81a
func Proxy(srvConn, cliConn *net.TCPConn) {
// channels to wait on the close event for each connection
serverClosed := make(chan struct{}, 1)
clientClosed := make(chan struct{}, 1)
go broker(srvConn, cliConn, clientClosed)
go broker(cliConn, srvConn, serverClosed)
// wait for one half of the proxy to exit, then trigger a shutdown of the
// other half by calling CloseRead(). This will break the read loop in the
// broker and allow us to fully close the connection cleanly without a
// "use of closed network connection" error.
var waitFor chan struct{}
select {
case <-clientClosed:
// the client closed first and any more packets from the server aren't
// useful, so we can optionally SetLinger(0) here to recycle the port
// faster.
srvConn.SetLinger(0)
srvConn.CloseRead()
waitFor = serverClosed
case <-serverClosed:
cliConn.CloseRead()
waitFor = clientClosed
}
// Wait for the other connection to close.
// This "waitFor" pattern isn't required, but gives us a way to track the
// connection and ensure all copies terminate correctly; we can trigger
// stats on entry and deferred exit of this function.
<-waitFor
}
// This does the actual data transfer.
// The broker only closes the Read side.
func broker(dst, src net.Conn, srcClosed chan struct{}) {
// We can handle errors in a finer-grained manner by inlining io.Copy (it's
// simple, and we drop the ReaderFrom or WriterTo checks for
// net.Conn->net.Conn transfers, which aren't needed). This would also let
// us adjust buffersize.
_, err := io.Copy(dst, src)
if err != nil {
log.Printf("Copy error: %s", err)
}
if err := src.Close(); err != nil {
log.Printf("Close error: %s", err)
}
srcClosed <- struct{}{}
}
It turned out that I had to restart the listener not only to close the connection. I've modified the broker function to reset the destNet listener if it can't write (i.e. writes 0 bytes) to src. I'm still not sure if this is the right way to do it (i.e. closing the listener seems bad in a multi-connections scenario as I guess I reset all the client connections dialing on that address) but so far this is the best I could do to fix it.
if n == 0 {
lrNewCh <- 1
}
Here is all the code. All the credit goes to #JimB
// Make a bridge between dstNet which is
// usually behind NAT and srcNet which is usually a client
// which wants to route the traffic though the NAT machine.
package main
import (
log "github.com/golang/glog"
"io"
"net"
)
// listen on the dstNet so that we can
// create a connection with the NAT client
var dstNet *net.TCPAddr = &net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 9000}
// listen on srcNet so that we can get traffic
// to forward to dstNet
var srcNet *net.TCPAddr = &net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 9001}
var errCh = make(chan error, 1)
// make a channel to send the reverse connections
var lrCh = make(chan *net.TCPConn, 1)
var lrNewCh = make(chan int, 1)
func listenDst() {
// Listen on the dstNet
lr, err := net.ListenTCP("tcp", dstNet)
if err != nil {
log.Error(err)
errCh <- err
return
}
// accept the connection
for {
lrConn, err := lr.AcceptTCP()
if err != nil {
log.Error(err)
//errCh <- err
//return
}
status := <-lrNewCh
log.Errorf("status request is %v", status)
if status == 1{
log.Errorf("we close and restart the listener and the connection")
if err = lrConn.Close(); err !=nil{
log.Error(err)
}
if err = lr.Close(); err !=nil{
log.Error(err)
}
lr, err = net.ListenTCP("tcp", dstNet)
if err != nil {
log.Error(err)
errCh <- err
return
}
lrConn, err = lr.AcceptTCP()
if err !=nil{
log.Error(err)
errCh <- err
}
}else{
log.Errorf("new connection on its way")
lrCh <- lrConn
}
// default:
// log.Errorf("accepting new connections")
}
}
func main() {
go func() {
for err := range errCh {
if err != nil {
panic(err)
}
}
}()
// listen for the nat server
go listenDst()
// listen for clients to connect
l, err := net.ListenTCP("tcp", srcNet)
if err != nil {
log.Error(err)
panic(err)
}
// accept the connection
for {
conn, err := l.AcceptTCP()
if err != nil {
log.Error(err)
panic(err)
}
// serve the connection
go func(conn *net.TCPConn) {
defer conn.Close()
lrNewCh <- 0
dst := <-lrCh
defer dst.Close()
proxy(dst, conn)
}(conn)
}
}
func proxy(srvConn, cliConn *net.TCPConn) {
// channels to wait on the close event for each connection
serverClosed := make(chan struct{}, 1)
clientClosed := make(chan struct{}, 1)
go broker(srvConn, cliConn, clientClosed)
go broker(cliConn, srvConn, serverClosed)
// wait for one half of the proxy to exit, then trigger a shutdown of the
// other half by calling CloseRead(). This will break the read loop in the
// broker and allow us to fully close the connection cleanly without a
// "use of closed network connection" error.
var waitFor chan struct{}
select {
case <-clientClosed:
// the client closed first and any more packets from the server aren't
// useful, so we can optionally SetLinger(0) here to recycle the port
// faster.
srvConn.SetLinger(0)
srvConn.CloseRead()
waitFor = serverClosed
case <-serverClosed:
cliConn.CloseRead()
waitFor = clientClosed
}
// Wait for the other connection to close.
// This "waitFor" pattern isn't required, but gives us a way to track the
// connection and ensure all copies terminate correctly; we can trigger
// stats on entry and deferred exit of this function.
<-waitFor
}
// This does the actual data transfer.
// The broker only closes the Read side.
func broker(dst, src net.Conn, srcClosed chan struct{}) {
// We can handle errors in a finer-grained manner by inlining io.Copy (it's
// simple, and we drop the ReaderFrom or WriterTo checks for
// net.Conn->net.Conn transfers, which aren't needed). This would also let
// us adjust buffersize.
n, err := io.Copy(dst, src)
log.Errorf(" %v bytes copied", n)
if err != nil {
log.Errorf("Copy error: %s", err)
// errCh <- err
}
if err := src.Close(); err != nil {
log.Errorf("Close error: %s", err)
errCh <- err
}
if n == 0 {
lrNewCh <- 1
}
srcClosed <- struct{}{}
}

Resources