redigo error log: write: connection reset by peer - go

Almost the same amount of time (point in time as redigo error log: write: connection reset by peer?), redis error log:
Client id=45183 addr=127.0.0.1:40420 fd=39 name= age=39706 idle=46 flags=N db=0 sub=8 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=16114 oll=528 omem=8545237 events=rw cmd=ping scheduled to be closed ASAP for overcoming of output buffer limits.
go error log
write tcp 127.0.0.1:40806->127.0.0.1:6379: write: connection reset by peer
Before that, the Go program didn't receive the subscription message for about 7 minutes. I presume it was a cache overflow caused by messages not being consumed.
The Redis client-output-buffer-limit is the default configuration.
The linux fd and connection count are normal, and I can't find of a reason for the unconsumable.
Here is my code:
server.go
func WaitFroMsg(ctx context.Context, pool *redis.Pool, onMessage func(channel string, data []byte) error, channel ...string) (err error) {
conn := pool.Get()
psc := redis.PubSubConn{Conn: conn}
if err := psc.Subscribe(redis.Args{}.AddFlat(channel)...); err != nil {
return err
}
done := make(chan error, 1)
go func() {
for {
switch n := psc.Receive().(type) {
case error:
done <- fmt.Errorf("redis pubsub receive err: %v", n)
return
case redis.Message:
if err = onMessage(n.Channel, n.Data); err != nil {
done <- err
return
}
case redis.Subscription:
if n.Count == 0 {
fmt.Println("all channels are unsubscribed", channel)
done <- nil
return
}
}
}
}()
const healthCheck = time.Minute
ticker := time.NewTicker(healthCheck)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err = psc.Ping(""); err != nil {
fmt.Println("healthCheck ", err, channel)
return err
}
case err := <-done:
return err
case <-ctx.Done():
if err := psc.Unsubscribe(); err != nil {
return fmt.Errorf("redis unsubscribe failed: %v", err)
}
return nil
}
}
}
pool.go
func NewPool(addr string, db int) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", addr)
if err != nil {
return nil, err
}
if _, err = c.Do("SELECT", db); err != nil {
c.Close()
return nil, err
}
return c, nil
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if time.Since(t) < time.Minute {
return nil
}
_, err := c.Do("PING")
fmt.Println("PING error", err)
return err
},
}
}

Related

redis pool.Get() takes a longer time to return connection in golang

This is the code I have written to get to create redisPool at compile time.
var RedisPool *redis.Pool //set at compile time
func RedisSetup() {
RedisPool = newRedisPool(serverUrl, password)
c := RedisPool.Get()
defer c.Close()
pong, err := redis.String(c.Do("PING"))
if err != nil{
fmt.Println("error while connecting redis ", err)
}
fmt.Println("Redis Ping:", pong)
}
func newRedisPool(server, password string) *redis.Pool {
pool := &redis.Pool{
MaxIdle: 10,
MaxActive: 100,
IdleTimeout: 5 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", server, redis.DialUseTLS(true), redis.DialTLSSkipVerify(true))
if err != nil {
log.Printf("ERROR: fail initializing the redis pool: %s", err.Error())
fmt.Println("server conn err", err)
return nil, err
}
if password != "" {
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
fmt.Println("auth err", err)
return nil, err
}
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
return pool
}
Now, this is the function i am using for doing redis operations like set or get.
func GetCacheValue(ctx context.Context, key string) (string, error) {
startTime := time.Now()
span, ctx := apm.StartSpan(ctx, "get cache " + key, "request")
defer span.End()
conn := apmredigo.Wrap(getPool(ctx)).WithContext(ctx)
val, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", err
}
redisPool.Close()
fmt.Println("\n redis get time : ", time.Since(startTime), "\n")
return val, nil
}
func getPool(ctx context.Context) redis.Conn {
startTime :=time.Now()
span, ctx := apm.StartSpan(ctx, "get connection ", "request")
defer span.End()
conn := RedisPool.Get()
fmt.Println("\nPool get time : ", time.Since(startTime), "\n")
return conn
}
This is the output i am getting, I have hit 3 request and got different different response time ,sometime it respond within 1ms or 2ms and sometime it takes around 100ms. Why so much difference? Can someone please help, is there anything I am missing?
console output 1 console output 2 apm server output

Golang Redis PubSub sends duplicate messages

I am trying to build a feature with the redis pub/sub implementation on golang
but for every client connected to the server (WebSocket) the number of messages published doubles.
func (c client) Subscribe() {
con := initRedis()
defer con.Close()
psc := redis.PubSubConn{Conn: con}
defer psc.Close()
psc.Subscribe(c.Channel)
for {
switch v := psc.Receive().(type) {
case redis.Message:
fmt.Printf("%s: message: %s\n", v.Channel, v.Data)
broadcast <- map[string]string{
"channel": v.Channel,
"message": string(v.Data),
}
case redis.Subscription:
fmt.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count)
case error:
log.Println(v)
psc.Unsubscribe()
}
}
}
func (m Message) Publish() {
c := initRedis()
defer c.Close()
reply, err := c.Do("PUBLISH", m.Channel, m.Msg)
if err != nil {
log.Println(err)
}
fmt.Println("Publishing", m, reply)
}
func initRedis() redis.Conn {
if err := godotenv.Load(); err != nil {
panic(err)
}
// fmt.Println(os.Getenv("REDIS_URL"))
c, err := redis.DialURL(os.Getenv("REDIS_URL"), redis.DialTLSSkipVerify(true))
if err != nil {
panic(err)
}
// defer c.Close()
return c
}
How do I stop the duplicates?
or what is causing the duplicates.

How I can close the connection by timeout if the client doesn't respond in 10 seconds?

I have code (I use https://github.com/fiorix/go-smpp):
// -----------------------------------------------
// handleConnection new clients.
// -----------------------------------------------
func (_srv *ServerSmpp) handleConnection(_cfg *ConfigSmpp, c *conn) {
defer c.Close()
if err := _srv.auth(_cfg, c); err != nil {
if err != io.EOF {
log.Printf("smpp_server: server auth failed: %s\n", err)
}
return
}
notify := make(chan error)
go func() {
for {
pb, err := c.Read()
if err != nil {
notify <- err
return
}
err = _srv.Handler(_srv.RemoteProvider, c, pb)
if err != nil {
fmt.Printf("%s\n", err)
notify <- err
return
}
}
}()
for {
select {
case err:= <-notify:
if io.EOF == err {
fmt.Printf("Smpp server (read): %s\n", err)
return
}
case <-time.After(time.Second * 10):
fmt.Printf("Client disconnected by timeout.\n")
return
}
}
}
Code for invoked handleConnection:
func (_srv *ServerSmpp) Serve(_cfg *ConfigSmpp) {
for {
client, err := _srv.NetListener.Accept()
if err != nil {
break
}
c := newConn(client)
go _srv.handleConnection(_cfg, c)
}
}
When this code work, the server disconnects all clients by timeout 10 sec, but how I can disconnect the client when it's doesn't work 10 sec?
Your client object seems to be a net.Conn,
choose a way to call client.SetReadDeadline() with the appropriate time.Time value before blocking on client.Read() :
c.client.SetDeadline( time.Now().Add(10 * time.Second )
pb, err := c.Read() { ...

Closing a redis subscription and ending the go routine when websocket connection closes

I'm pushing events from a redis subscription to a client who is connected via websocket. I'm having trouble unsubscribing and exiting the redis go routine when the client disconnects the websocket.
Inspired by this post, here's what I have thus far. I'm able to receive subscription events and send messages to the client via websocket, but when the client closes the websocket and the defer close(done) code fires, my case b, ok := <-done: doesn't fire. It seems to be overloaded by the default case???
package api
import (
...
"github.com/garyburd/redigo/redis"
"github.com/gorilla/websocket"
)
func wsHandler(w http.ResponseWriter, r *http.Request) {
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
HandleError(w, err)
return
}
defer conn.Close()
done := make(chan bool)
defer close(done)
for {
var req WSRequest
err := conn.ReadJSON(&req)
if err != nil {
HandleWSError(conn, err)
return
}
defer conn.Close()
go func(done chan bool, req *WSRequest, conn *websocket.Conn) {
rc := redisPool.Get()
defer rc.Close()
psc := redis.PubSubConn{Conn: rc}
if err := psc.PSubscribe(req.chanName); err != nil {
HandleWSError(conn, err)
return
}
defer psc.PUnsubscribe()
for {
select {
case b, ok := <-done:
if !ok || b == true {
return
}
default:
switch v := psc.Receive().(type) {
case redis.PMessage:
err := handler(conn, req, v)
if err != nil {
HandleWSError(conn, err)
}
case redis.Subscription:
log.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count)
case error:
log.Printf("error in redis subscription; err:\n%v\n", v)
HandleWSError(conn, v)
default:
// do nothing...
log.Printf("unknown redis subscription event type; %s\n", reflect.TypeOf(v))
}
}
}
}(done, &req, conn)
}
}
Make these changes to break out of the read loop when done serving the websocket connection:
Maintain a slice of the Redis connections created for this websocket connection.
Unsubscribe all connections when done.
Modify the read loop to return when the subscription count is zero.
Here's the code:
func wsHandler(w http.ResponseWriter, r *http.Request) {
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
HandleError(w, err)
return
}
defer conn.Close()
// Keep slice of all connections. Unsubscribe all connections on exit.
var pscs []redis.PubSubConn
defer func() {
for _, psc := range rcs {
psc.Unsubscribe() // unsubscribe with no args unsubs all channels
}
}()
for {
var req WSRequest
err := conn.ReadJSON(&req)
if err != nil {
HandleWSError(conn, err)
return
}
rc := redisPool.Get()
psc := redis.PubSubConn{Conn: rc}
pscs = append(pscs, psc)
if err := psc.PSubscribe(req.chanName); err != nil {
HandleWSError(conn, err)
return
}
go func(req *WSRequest, conn *websocket.Conn) {
defer rc.Close()
for {
switch v := psc.Receive().(type) {
case redis.PMessage:
err := handler(conn, req, v)
if err != nil {
HandleWSError(conn, err)
}
case redis.Subscription:
log.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count)
if v.Count == 0 {
return
}
case error:
log.Printf("error in redis subscription; err:\n%v\n", v)
HandleWSError(conn, v)
default:
// do nothing...
log.Printf("unknown redis subscription event type; %s\n", reflect.TypeOf(v))
}
}
}(&req, conn)
}
}
The code in the question and this answer dial multiple Redis connections for each websocket client. A more typical and scalable approach is to share a single Redis pubsub connection across multiple clients. The typical approach may be appropriate for your application given the high-level description, but I am still unsure of what you are trying to do given the code in the question.

SETEX error - "Use of closed network connection"

I'm using the following code to execute a SET and EXPIRE from my Go app.
_, err = C.Cache.Do("SETEX", key, 3600, data)
but I've started to get an error: Use of closed network connection. I use Gary Burd's Redigo package and RedisLabs.
My code to connect to Redis is:
//Connect to cache (Redis)
cache, err := connectToCache()
if err != nil {
log.Printf("Cache connection settings are invalid")
os.Exit(1)
}
defer cache.Close()
func connectToCache() (redis.Conn, error) {
cache, err := redis.Dial("tcp", CACHE_URI)
if err != nil {
return nil, err
}
_, err = cache.Do("AUTH", CACHE_AUTH)
if err != nil {
cache.Close()
return nil, err
}
return cache, nil
}
You can use a redis.Pool to manage multiple connections, check that idle connections are alive, and get new connections automatically. You can also do the AUTH step automatically when dialing a new connection:
func newPool(server, password string) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func () (redis.Conn, error) {
c, err := redis.Dial("tcp", server)
if err != nil {
return nil, err
}
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
var (
pool *redis.Pool
redisServer = flag.String("redisServer", ":6379", "")
redisPassword = flag.String("redisPassword", "", "")
)
func main() {
flag.Parse()
pool = newPool(*redisServer, *redisPassword)
...
}

Resources