SETEX error - "Use of closed network connection" - go

I'm using the following code to execute a SET and EXPIRE from my Go app.
_, err = C.Cache.Do("SETEX", key, 3600, data)
but I've started to get an error: Use of closed network connection. I use Gary Burd's Redigo package and RedisLabs.
My code to connect to Redis is:
//Connect to cache (Redis)
cache, err := connectToCache()
if err != nil {
log.Printf("Cache connection settings are invalid")
os.Exit(1)
}
defer cache.Close()
func connectToCache() (redis.Conn, error) {
cache, err := redis.Dial("tcp", CACHE_URI)
if err != nil {
return nil, err
}
_, err = cache.Do("AUTH", CACHE_AUTH)
if err != nil {
cache.Close()
return nil, err
}
return cache, nil
}

You can use a redis.Pool to manage multiple connections, check that idle connections are alive, and get new connections automatically. You can also do the AUTH step automatically when dialing a new connection:
func newPool(server, password string) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func () (redis.Conn, error) {
c, err := redis.Dial("tcp", server)
if err != nil {
return nil, err
}
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
var (
pool *redis.Pool
redisServer = flag.String("redisServer", ":6379", "")
redisPassword = flag.String("redisPassword", "", "")
)
func main() {
flag.Parse()
pool = newPool(*redisServer, *redisPassword)
...
}

Related

SMTP client using a remote SOCKS5/proxy in Go

I am trying to create an SMTP client that uses a proxy connection, SOCKS5.
When I use a local host proxy the code successfully creates an SMTP client.
When I try to use a remote proxy, I am getting a TTL expired. I am also getting an EOF error when trying to use a different proxy connection.
I have set up a proxy server in my localhost, socks5://dante:maluki#127.0.0.1:1080
I have also set up an identical proxy server on my remote VM, socks5://dante:maluki#35.242.186.23:1080
package main
import (
"errors"
"log"
"net"
"net/smtp"
"net/url"
"time"
"golang.org/x/net/idna"
"golang.org/x/net/proxy"
)
const (
smtpTimeout = time.Second * 60
smtpPort = ":25"
)
func init() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
}
func main() {
// The code works when I use a localhost proxy
// socks5://dante:maluki#127.0.0.1:1080
client, err := newSMTPClient("gmail.com", "socks5://dante:maluki#35.242.186.23:1080")
if err != nil {
log.Println(err)
return
}
log.Println(client)
}
// establishProxyConnection connects to the address on the named network address
// via proxy protocol
func establishProxyConnection(addr, proxyURI string) (net.Conn, error) {
// return socks.Dial(proxyURI)("tcp", addr)
u, err := url.Parse(proxyURI)
if err != nil {
log.Println(err)
return nil, err
}
var iface proxy.Dialer
if u.User != nil {
auth := proxy.Auth{}
auth.User = u.User.Username()
auth.Password, _ = u.User.Password()
iface, err = proxy.SOCKS5("tcp", u.Host, &auth, &net.Dialer{Timeout: 30 * time.Second})
if err != nil {
log.Println(err)
return nil, err
}
} else {
iface, err = proxy.SOCKS5("tcp", u.Host, nil, proxy.FromEnvironment())
if err != nil {
log.Println(err)
return nil, err
}
}
dialfunc := iface.Dial
return dialfunc("tcp", addr)
}
// newSMTPClient generates a new available SMTP client
func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) {
domain = domainToASCII(domain)
mxRecords, err := net.LookupMX(domain)
if err != nil {
log.Println(err)
return nil, err
}
if len(mxRecords) == 0 {
return nil, errors.New("No MX records found")
}
// Attempt to connect to SMTP servers
for _, r := range mxRecords {
// Simplified to make the code short
addr := r.Host + smtpPort
c, err := dialSMTP(addr, proxyURI)
if err != nil {
log.Println(err)
continue
}
return c, err
}
return nil, errors.New("failed to created smtp.Client")
}
// dialSMTP is a timeout wrapper for smtp.Dial. It attempts to dial an
// SMTP server (socks5 proxy supported) and fails with a timeout if timeout is reached while
// attempting to establish a new connection
func dialSMTP(addr, proxyURI string) (*smtp.Client, error) {
// Channel holding the new smtp.Client or error
ch := make(chan interface{}, 1)
// Dial the new smtp connection
go func() {
var conn net.Conn
var err error
conn, err = establishProxyConnection(addr, proxyURI)
if err != nil {
log.Println(err)
}
if err != nil {
ch <- err
return
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
log.Println(err)
}
client, err := smtp.NewClient(conn, host)
log.Println(client)
if err != nil {
log.Println(err)
ch <- err
return
}
ch <- client
}()
// Retrieve the smtp client from our client channel or timeout
select {
case res := <-ch:
switch r := res.(type) {
case *smtp.Client:
return r, nil
case error:
return nil, r
default:
return nil, errors.New("Unexpected response dialing SMTP server")
}
case <-time.After(smtpTimeout):
return nil, errors.New("Timeout connecting to mail-exchanger")
}
}
// domainToASCII converts any internationalized domain names to ASCII
// reference: https://en.wikipedia.org/wiki/Punycode
func domainToASCII(domain string) string {
asciiDomain, err := idna.ToASCII(domain)
if err != nil {
return domain
}
return asciiDomain
}

How do I make it such that a TCP connection will timeout if the connection doesn't receive a response every second?

I'm trying to create a TCP server that will timeout if the client does not respond within the span of every second.
I tried:
func main() {
listener, err := net.Listen("tcp", "localhost:8000")
if err != nil {
log.Fatal(err)
}
for {
conn, err := listener.Accept()
conn.SetDeadline(time.Now().Add(timeout))
if err != nil {
log.Print(err)
}
go handleConn(conn)
}
}
where the timeout is a single second but the disconnects immediately, not even waiting for a reply.
What you want can be achieved by setting socket options on your listener. Tweak the values as per your needs
Note that this is its own KeepAlive and does not depend on incoming/outgoing data by application
func enableTCPKeepAlive(listener *net.TCPListener) error {
rawConn, err := listener.SyscallConn()
if err != nil {
return err
}
cfg := config.TLSServerConfig()
rawConn.Control(
func(fdPtr uintptr) {
// got socket file descriptor. Setting parameters.
fd := int(fdPtr)
//Idle time before sending probe.
err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, cfg.TCPKAIdleTime)
if err != nil {
return err
}
//Number of probes.
err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, cfg.TCPKANumProbes)
if err != nil {
return err
}
//Wait time after an unsuccessful probe.
err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, cfg.TCPKAInterval)
if err != nil {
return err
}
// go syscall doesn't have the constant 0x12 (18) for TCP_USER_TIMEOUT.
// 0x12 value referenced from linux kernel source code header:
// include/uapi/linux/tcp.h
err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, 0x12, cfg.TCPKAUserTimeout)
if err != nil {
return err
}
})
return nil
}
There are more options available than the ones I have mentioned above.
Call this function on your listener before the for loop.
func main() {
listener, err := net.Listen("tcp", "localhost:8000")
if err != nil {
log.Fatal(err)
}
err = enableTCPKeepAlive(listener)
if err != nil {
log.Fatal(err)
}
for {
conn, err := listener.Accept()
conn.SetDeadline(time.Now().Add(timeout))
if err != nil {
log.Print(err)
}
go handleConn(conn)
}
}
The problem is almost always in code that is not posted here. The function obviously works like a charme:
package main
import (
"crypto/rand"
"log"
"net"
"time"
)
func main() {
listener, err := net.Listen("tcp", "localhost:8000")
if err != nil {
log.Fatal(err)
}
go func() {
for {
conn, err := listener.Accept()
if err != nil {
log.Print(err)
return
}
go func(c net.Conn) {
defer c.Close()
conn.SetDeadline(time.Now().Add(2 * time.Second))
if err != nil {
log.Print(err)
return
}
buf := make([]byte, 1<<19) // 512 KB
for {
_, err := conn.Read(buf)
if err != nil {
log.Print(err)
break
}
}
}(conn)
}
}()
payload := make([]byte, 1<<20)
_, err = rand.Read(payload) // generate a random payload
if err != nil {
log.Print(err)
}
conn, err := net.Dial("tcp", listener.Addr().String())
if err != nil {
log.Fatal(err)
}
log.Println("Connected to server.")
time.Sleep(5 * time.Second)
_, err = conn.Write(payload)
if err != nil {
log.Print(err)
}
listener.Close()
}

How do I set a timeout for a google translation?

https://cloud.google.com/translate/docs/samples/translate-text-with-model?hl=zh-cn#translate_text_with_model-go
I'm using the example in the link.When I turn off the proxy.It seems to take 30s to time out.How do I set the timeout duration?And is there an example?
Replace 'context.Background()' with 'context.WithTimeout()' seems doesn't work.
func main() {
fmt.Println("start..")
t := "The Go Gopher is cute"
now := time.Now()
r, err := translateTextWithModel("zh-CN", t, "nmt")
fmt.Println(time.Now().Sub(now).Milliseconds(), "ms")
fmt.Println(t, "-->", r)
fmt.Println("err:", err)
fmt.Println("end..")
}
func translateTextWithModel(targetLanguage, text, model string) (string, error) {
// targetLanguage := "ja"
// text := "The Go Gopher is cute"
// model := "nmt"
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
lang, err := language.Parse(targetLanguage)
if err != nil {
return "", fmt.Errorf("language.Parse: %v", err)
}
client, err := translate.NewClient(ctx)
if err != nil {
return "", fmt.Errorf("translate.NewClient: %v", err)
}
defer client.Close()
resp, err := client.Translate(ctx, []string{text}, lang, &translate.Options{
Model: model, // Either "nmt" or "base".
})
if err != nil {
return "", fmt.Errorf("Translate: %v", err)
}
if len(resp) == 0 {
return "", nil
}
return resp[0].Text, nil
}
When turning off the proxy,sometimes like.
As I see the library provides possibility to change settings by providing options into constructor function translate.NewClient(ctx context.Context, opts ...option.ClientOption).
You can try to change http.Client timeout by creating new one:
// timeout that you need to set
httpClientTimeout := time.Minute * 5
// create http client with custom timeout
httpClient := &http.Client{
Timeout: httpClientTimeout,
}
// inject new http client into translate client
client, err := translate.NewClient(ctx, option.WithHTTPClient(httpClient))
if err != nil {
return "", fmt.Errorf("translate.NewClient: %v", err)
}
defer client.Close()
Hope it helpful.
Source documentation

redis pool.Get() takes a longer time to return connection in golang

This is the code I have written to get to create redisPool at compile time.
var RedisPool *redis.Pool //set at compile time
func RedisSetup() {
RedisPool = newRedisPool(serverUrl, password)
c := RedisPool.Get()
defer c.Close()
pong, err := redis.String(c.Do("PING"))
if err != nil{
fmt.Println("error while connecting redis ", err)
}
fmt.Println("Redis Ping:", pong)
}
func newRedisPool(server, password string) *redis.Pool {
pool := &redis.Pool{
MaxIdle: 10,
MaxActive: 100,
IdleTimeout: 5 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", server, redis.DialUseTLS(true), redis.DialTLSSkipVerify(true))
if err != nil {
log.Printf("ERROR: fail initializing the redis pool: %s", err.Error())
fmt.Println("server conn err", err)
return nil, err
}
if password != "" {
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
fmt.Println("auth err", err)
return nil, err
}
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
return pool
}
Now, this is the function i am using for doing redis operations like set or get.
func GetCacheValue(ctx context.Context, key string) (string, error) {
startTime := time.Now()
span, ctx := apm.StartSpan(ctx, "get cache " + key, "request")
defer span.End()
conn := apmredigo.Wrap(getPool(ctx)).WithContext(ctx)
val, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", err
}
redisPool.Close()
fmt.Println("\n redis get time : ", time.Since(startTime), "\n")
return val, nil
}
func getPool(ctx context.Context) redis.Conn {
startTime :=time.Now()
span, ctx := apm.StartSpan(ctx, "get connection ", "request")
defer span.End()
conn := RedisPool.Get()
fmt.Println("\nPool get time : ", time.Since(startTime), "\n")
return conn
}
This is the output i am getting, I have hit 3 request and got different different response time ,sometime it respond within 1ms or 2ms and sometime it takes around 100ms. Why so much difference? Can someone please help, is there anything I am missing?
console output 1 console output 2 apm server output

redigo error log: write: connection reset by peer

Almost the same amount of time (point in time as redigo error log: write: connection reset by peer?), redis error log:
Client id=45183 addr=127.0.0.1:40420 fd=39 name= age=39706 idle=46 flags=N db=0 sub=8 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=16114 oll=528 omem=8545237 events=rw cmd=ping scheduled to be closed ASAP for overcoming of output buffer limits.
go error log
write tcp 127.0.0.1:40806->127.0.0.1:6379: write: connection reset by peer
Before that, the Go program didn't receive the subscription message for about 7 minutes. I presume it was a cache overflow caused by messages not being consumed.
The Redis client-output-buffer-limit is the default configuration.
The linux fd and connection count are normal, and I can't find of a reason for the unconsumable.
Here is my code:
server.go
func WaitFroMsg(ctx context.Context, pool *redis.Pool, onMessage func(channel string, data []byte) error, channel ...string) (err error) {
conn := pool.Get()
psc := redis.PubSubConn{Conn: conn}
if err := psc.Subscribe(redis.Args{}.AddFlat(channel)...); err != nil {
return err
}
done := make(chan error, 1)
go func() {
for {
switch n := psc.Receive().(type) {
case error:
done <- fmt.Errorf("redis pubsub receive err: %v", n)
return
case redis.Message:
if err = onMessage(n.Channel, n.Data); err != nil {
done <- err
return
}
case redis.Subscription:
if n.Count == 0 {
fmt.Println("all channels are unsubscribed", channel)
done <- nil
return
}
}
}
}()
const healthCheck = time.Minute
ticker := time.NewTicker(healthCheck)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err = psc.Ping(""); err != nil {
fmt.Println("healthCheck ", err, channel)
return err
}
case err := <-done:
return err
case <-ctx.Done():
if err := psc.Unsubscribe(); err != nil {
return fmt.Errorf("redis unsubscribe failed: %v", err)
}
return nil
}
}
}
pool.go
func NewPool(addr string, db int) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", addr)
if err != nil {
return nil, err
}
if _, err = c.Do("SELECT", db); err != nil {
c.Close()
return nil, err
}
return c, nil
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if time.Since(t) < time.Minute {
return nil
}
_, err := c.Do("PING")
fmt.Println("PING error", err)
return err
},
}
}

Resources