Golang and redis newbie. I have a redis cluster in amazon with 9 nodes. 3 master and for each master we have 2 slaves. Total nine nodes. Not using RetryConn returns a MOVED node error and using it returns a null.
Is it important to give the IP address for all the nodes? IS it ok to give the host name that amz gives when we create the cluster.
cluster = redisc.Cluster{
StartupNodes: []string{"*****"},
DialOptions: []redis.DialOption{redis.DialConnectTimeout(5 *
time.Minute)},
CreatePool: createPool,
}
if err := cluster.Refresh(); err != nil {
log.Fatalf("Refresh failed: %v", err)
}else{
log.Println("Refresh worked")
}
func createPool(addr string, opts ...redis.DialOption) (*redis.Pool,
error) {
return &redis.Pool{
MaxIdle: 5,
MaxActive: 10,
IdleTimeout: time.Minute,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", "***", opts...)
if err != nil {
log.Panic(err)
}
}, nil
}
func RedisConnection() redis.Conn {
// grab a connection from the pool
conn := cluster.Get()
if(conn != nil){
rc, err := redisc.RetryConn(conn, 9, 1*time.Second)
if(err ==nil){
return rc
}
return nil
}
}
What am I doing wrong?
Also how do we binding the conn needed?Any insight would be very helpful
Related
This is the code I have written to get to create redisPool at compile time.
var RedisPool *redis.Pool //set at compile time
func RedisSetup() {
RedisPool = newRedisPool(serverUrl, password)
c := RedisPool.Get()
defer c.Close()
pong, err := redis.String(c.Do("PING"))
if err != nil{
fmt.Println("error while connecting redis ", err)
}
fmt.Println("Redis Ping:", pong)
}
func newRedisPool(server, password string) *redis.Pool {
pool := &redis.Pool{
MaxIdle: 10,
MaxActive: 100,
IdleTimeout: 5 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", server, redis.DialUseTLS(true), redis.DialTLSSkipVerify(true))
if err != nil {
log.Printf("ERROR: fail initializing the redis pool: %s", err.Error())
fmt.Println("server conn err", err)
return nil, err
}
if password != "" {
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
fmt.Println("auth err", err)
return nil, err
}
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
return pool
}
Now, this is the function i am using for doing redis operations like set or get.
func GetCacheValue(ctx context.Context, key string) (string, error) {
startTime := time.Now()
span, ctx := apm.StartSpan(ctx, "get cache " + key, "request")
defer span.End()
conn := apmredigo.Wrap(getPool(ctx)).WithContext(ctx)
val, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", err
}
redisPool.Close()
fmt.Println("\n redis get time : ", time.Since(startTime), "\n")
return val, nil
}
func getPool(ctx context.Context) redis.Conn {
startTime :=time.Now()
span, ctx := apm.StartSpan(ctx, "get connection ", "request")
defer span.End()
conn := RedisPool.Get()
fmt.Println("\nPool get time : ", time.Since(startTime), "\n")
return conn
}
This is the output i am getting, I have hit 3 request and got different different response time ,sometime it respond within 1ms or 2ms and sometime it takes around 100ms. Why so much difference? Can someone please help, is there anything I am missing?
console output 1 console output 2 apm server output
Almost the same amount of time (point in time as redigo error log: write: connection reset by peer?), redis error log:
Client id=45183 addr=127.0.0.1:40420 fd=39 name= age=39706 idle=46 flags=N db=0 sub=8 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=16114 oll=528 omem=8545237 events=rw cmd=ping scheduled to be closed ASAP for overcoming of output buffer limits.
go error log
write tcp 127.0.0.1:40806->127.0.0.1:6379: write: connection reset by peer
Before that, the Go program didn't receive the subscription message for about 7 minutes. I presume it was a cache overflow caused by messages not being consumed.
The Redis client-output-buffer-limit is the default configuration.
The linux fd and connection count are normal, and I can't find of a reason for the unconsumable.
Here is my code:
server.go
func WaitFroMsg(ctx context.Context, pool *redis.Pool, onMessage func(channel string, data []byte) error, channel ...string) (err error) {
conn := pool.Get()
psc := redis.PubSubConn{Conn: conn}
if err := psc.Subscribe(redis.Args{}.AddFlat(channel)...); err != nil {
return err
}
done := make(chan error, 1)
go func() {
for {
switch n := psc.Receive().(type) {
case error:
done <- fmt.Errorf("redis pubsub receive err: %v", n)
return
case redis.Message:
if err = onMessage(n.Channel, n.Data); err != nil {
done <- err
return
}
case redis.Subscription:
if n.Count == 0 {
fmt.Println("all channels are unsubscribed", channel)
done <- nil
return
}
}
}
}()
const healthCheck = time.Minute
ticker := time.NewTicker(healthCheck)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err = psc.Ping(""); err != nil {
fmt.Println("healthCheck ", err, channel)
return err
}
case err := <-done:
return err
case <-ctx.Done():
if err := psc.Unsubscribe(); err != nil {
return fmt.Errorf("redis　unsubscribe failed: %v", err)
}
return nil
}
}
}
pool.go
func NewPool(addr string, db int) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", addr)
if err != nil {
return nil, err
}
if _, err = c.Do("SELECT", db); err != nil {
c.Close()
return nil, err
}
return c, nil
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if time.Since(t) < time.Minute {
return nil
}
_, err := c.Do("PING")
fmt.Println("PING error", err)
return err
},
}
}
I have an application that makes about 400 reads per seconds and 100 writes per second to redis (hosted on redislabs). The application is using github.com/garyburd/redigo package as a redis proxy.
I have two functions which are the only ones being used to read and write:
func getCachedVPAIDConfig(key string) chan *cachedVPAIDConfig {
c := make(chan *cachedVPAIDConfig)
go func() {
p := pool.Get()
defer p.Close()
switch p.Err() {
case nil:
item, err := redis.Bytes(p.Do("GET", key))
if err != nil {
c <- &cachedVPAIDConfig{nil, err}
return
}
c <- &cachedVPAIDConfig{item, nil}
default:
c <- &cachedVPAIDConfig{nil, p.Err()}
return
}
}()
return c
}
func setCachedVPAIDConfig(key string, j []byte) chan error {
c := make(chan error)
go func() {
p := pool.Get()
defer p.Close()
switch p.Err() {
case nil:
_, err := p.Do("SET", key, j)
if err != nil {
c <- err
return
}
c <- nil
default:
c <- p.Err()
return
}
}()
return c
}
As you can see, I'm using the recommended connection pooling mechanism (http://godoc.org/github.com/garyburd/redigo/redis#Pool).
I'm calling these functions on every http request an endpoint on the application is getting. The problem is: once the application starts getting requests, it immediately starts throwing the error
dial tcp 54.160.xxx.xx:yyyy: connect: cannot assign requested address
(54.160.xxx.xx:yyyy is the redis host)
I see on redis that there are only about 600 connections when this starts to happen, which doesn't sound like a lot.
I tried playing with the MaxActive setting of the pool, setting it anywhere between 1000 and 50K, but the result is the same.
Any ideas?
EDIT
Here's my pool initialization code (doing this in func init):
pool = redis.Pool{
MaxActive: 1000, // note: I tried changing this to 50K, result the same
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", redisHost)
if err != nil {
return nil, err
}
if _, err := c.Do("AUTH", redisPassword); err != nil {
c.Close()
return nil, err
}
return c, err
},
}
Edit 2:
Issue solved by applying the stuff suggested in the answer below!
New code for pool init:
pool = redis.Pool{
MaxActive: 500,
MaxIdle: 500,
IdleTimeout: 5 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", redisHost, 100*time.Millisecond, 100*time.Millisecond, 100*time.Millisecond)
if err != nil {
return nil, err
}
if _, err := c.Do("AUTH", redisPassword); err != nil {
c.Close()
return nil, err
}
return c, err
},
}
This new init makes it so that the get and set timeouts are handled by redigo internally, so I no longer need to return a channel on the getCachedVPAIDConfig and setCachedVPAIDConfig funcs. This is how they look now:
func setCachedVPAIDConfig(key string, j []byte) error {
p := pool.Get()
switch p.Err() {
case nil:
_, err := p.Do("SET", key, j)
p.Close()
return err
default:
p.Close()
return p.Err()
}
}
func getCachedVPAIDConfig(key string) ([]byte, error) {
p := pool.Get()
switch p.Err() {
case nil:
item, err := redis.Bytes(p.Do("GET", key))
p.Close()
return item, err
default:
p.Close()
return nil, p.Err()
}
}
You're closing the connection after sending on the channels, if the channel is blocking you're not closing connections, which would result in the error you're seeing. so don't just defer, close the connection explicitly.
I don't think it's the problem but a good idea regardless - set a timeout on your connections with DialTimeout.
Make sure you have a proper TestOnBorrow function to get rid of dead connections, especially if you have timeout. I usually do a PING if the connection has been idle for more than 3 seconds (the function receives the idle time as a parameter)
Try setting MaxIdle to a larger number as well, I remember having problems with pooling that were resolved by increasing that parameter in the pool.
I'm using the following code to execute a SET and EXPIRE from my Go app.
_, err = C.Cache.Do("SETEX", key, 3600, data)
but I've started to get an error: Use of closed network connection. I use Gary Burd's Redigo package and RedisLabs.
My code to connect to Redis is:
//Connect to cache (Redis)
cache, err := connectToCache()
if err != nil {
log.Printf("Cache connection settings are invalid")
os.Exit(1)
}
defer cache.Close()
func connectToCache() (redis.Conn, error) {
cache, err := redis.Dial("tcp", CACHE_URI)
if err != nil {
return nil, err
}
_, err = cache.Do("AUTH", CACHE_AUTH)
if err != nil {
cache.Close()
return nil, err
}
return cache, nil
}
You can use a redis.Pool to manage multiple connections, check that idle connections are alive, and get new connections automatically. You can also do the AUTH step automatically when dialing a new connection:
func newPool(server, password string) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func () (redis.Conn, error) {
c, err := redis.Dial("tcp", server)
if err != nil {
return nil, err
}
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
var (
pool *redis.Pool
redisServer = flag.String("redisServer", ":6379", "")
redisPassword = flag.String("redisPassword", "", "")
)
func main() {
flag.Parse()
pool = newPool(*redisServer, *redisPassword)
...
}
using redigo, I create a pool, something like so:
&redis.Pool{
MaxIdle: 80,
MaxActive: 12000, // max number of connections
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", host+":"+port)
if err != nil {
panic(err.Error())
}
return c, err
}
the problem I have is that for each time I get a new connection, I need to set the db, as I use different db's of redis since I host a number of sites on the VPS.
So, something like this:
conn := pool.Get()
defer conn.Close()
conn.Do("SELECT", dbNumber) //this is the call I want to avoid
Having to select the db each time I work with redis seems redundant and also poses a problem since I use it for sessions i.e. having code that is not mine working with my redis connection from my pool makes it "impossible" to set the correct db for it.
What I would like to do is to set the dbno for the pool so that whenever somebody asks for a new connection from the pool, it comes with the correct db already set i.e. not setting it explicitly each time.
How did you solve this in your applications?
Thanks.
You can use redis.DialOption: redis.DialDatabase, redis.DialPassword
conn, err := redis.Dial("tcp", "127.0.0.1:6379", redis.DialDatabase(1))
if err != nil {
panic(err)
}
defer conn.Close()
Select the database in your dial function:
&redis.Pool{
MaxIdle: 80,
MaxActive: 12000, // max number of connections
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", host+":"+port)
if err != nil {
return nil, err
}
_, err := c.Do("SELECT", dbNum)
if err != nil {
c.Close()
return nil, err
}
return c, nil
}
Also, return the error from dial instead of panicking.
If these libs don't support it, then you have two options:
submit a patch to automate this (the python lib does that, but be careful when keeping the state).
Wrap your redis pool with your own custom pool that automates this, something like (untested code, but you'll get the idea):
// a pool embedding the original pool and adding adbno state
type DbnoPool struct {
redis.Pool
dbno int
}
// "overriding" the Get method
func (p *DbnoPool)Get() Connection {
conn := p.Pool.Get()
conn.Do("SELECT", p.dbno)
return conn
}
pool := &DbnoPool {
redis.Pool{
MaxIdle: 80,
MaxActive: 12000, // max number of connections
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", host+":"+port)
if err != nil {
panic(err.Error())
}
return c, err
},
3, // the db number
}
//now you call it normally
conn := pool.Get()
defer conn.Close()
Best way is to use DialOptions like DialDatabase:
redisPool = &redis.Pool{
MaxIdle: AppConfig.DefaultInt("RedisMaxPool", 10),
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial(
"tcp",
AppConfig.DefaultString("RedisPath", ":6379"),
redis.DialDatabase(AppConfig.DefaultInt("RedisDB", 1)),
)
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}