Golang Redis PubSub timeout - go

So far I've been doing this:
import (
_redis "gopkg.in/redis.v3"
"strconv"
"time"
)
type Redis struct {
Connector *_redis.Client
PubSub *_redis.PubSub
}
var redis *Redis = nil
func NewRedis() bool {
if redis == nil {
redis = new(Redis)
redis.Connector = _redis.NewClient(&_redis.Options{
Addr: config.RedisHostname + ":" + strconv.FormatInt(config.RedisPort, 10),
Password: "",
DB: 0,
})
Logger.Log(nil, "Connected to Redis")
err := redis.Init()
if err != nil {
Logger.Fatal(nil, "Cannot setup Redis:", err.Error())
return false
}
return true
}
return false
}
func (this *Redis) Init() error {
pubsub, err := this.Connector.Subscribe("test")
if err != nil {
return err
}
defer pubsub.Close()
this.PubSub = pubsub
for {
msgi, err := this.PubSub.ReceiveTimeout(100 * time.Millisecond)
if err != nil {
Logger.Error(nil, "PubSub error:", err.Error())
err = this.PubSub.Ping("")
if err != nil {
Logger.Error(nil, "PubSub failure:", err.Error())
break
}
continue
}
switch msg := msgi.(type) {
case *_redis.Message:
Logger.Log(nil, "Received", msg.Payload, "on channel", msg.Channel)
}
}
return nil
}
My Connector is a redis.Client, it's working because I was able to publish messages as well.
When I run my program, I get the following error:
PubSub error: WSARecv tcp 127.0.0.1:64505: i/o timeout
Do you have any idea of what I'm doing wrong ? I'm using this package: https://github.com/go-redis/redis

Some things to note:
(implementation detail) When redis goes into PubSub mode, the only thing that happens on that socket afterwards is PubSub events, which is why PubSub in go-redis is abstracted into its own type
A PubSub client can potentially subscribe to multiple topics in a single subscriber, hence why there are subscribe/unsubscribe events throughout.
the interface has both Receive() and ReceiveTimeout(duration) methods, both of which return the next event on the wire; which can be subscribe/unsubscribe events and message events; (you don't necessarily know which) the only difference between them that Receive blocks forever until there's a new message, and ReceiveTimeout will error on timeout.
With that in mind, unless you have messages far more than 10/second consistently (in other words, <100 milliseconds between messages), it's inefficient to use that short of a timeout; and I'd argue that due to golang having goroutines, you should almost never use ReceiveTimeout for real applications, or use a sufficiently long timeout like a minute.
with that in mind, your receive loop should look like:
for {
msgi, err := this.PubSub.Receive()
if err != nil {
Logger.Error(nil, "PubSub error:", err.Error())
return err
}
switch msg := msgi.(type) {
case *_redis.Message:
Logger.Log(nil, "Received", msg.Payload, "on channel", msg.Channel)
default:
Logger.Log(nil, "Got control message", msg)
}
}
If your application really warranted using a timeout, then you should use a type assertion to assert the *net.OpError that signifies a timeout and distinguish it from other more serious errors.

Related

automatic gRPC unix reconnect after EOF

I have an application (let's call it client) connecting to another process (let's call it server) on the same machine via gRPC. The communication goes over unix socket.
If server is restarted, my client gets an EOF and does not re-establish the connection, although I expected the clientConn to handle the reconnection automatically.
Why isn't the dialer taking care of the reconnection?
I expect it to do so with the backoff params I passed.
Below some pseudo-MWE.
Run establish the initial connection, then spawns goroutineOne
goroutineOne waits for the connection to be ready and delegates the send to fooUpdater
fooUpdater streams the data, or returns in case of errors
for waitUntilReady I used the pseudo-code referenced by this answer to get a new stream.
func main() {
go func() {
if err := Run(ctx); err != nil {
log.Errorf("connection error: %v", err)
}
ctxCancel()
}()
// some wait logic
}
func Run(ctx context.Context) {
backoffConfig := backoff.Config{
BaseDelay: time.Duration(1 * time.Second),
Multiplier: backoff.DefaultConfig.Multiplier,
Jitter: backoff.DefaultConfig.Jitter,
MaxDelay: time.Duration(120 * time.Second),
}
myConn, err := grpc.DialContext(ctx,
"/var/run/foo.bar",
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithConnectParams(grpc.ConnectParams{Backoff: backoffConfig, MinConnectTimeout: time.Duration(1 * time.Second)}),
grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
d := net.Dialer{}
c, err := d.DialContext(ctx, "unix", addr)
if err != nil {
return nil, fmt.Errorf("connection to unix://%s failed: %w", addr, err)
}
return c, nil
}),
)
if err != nil {
return fmt.Errorf("could not establish socket for foo: %w", err)
}
defer myConn.Close()
return goroutineOne()
}
func goroutineOne() {
reconnect := make(chan struct{})
for {
if ready := waitUntilReady(ctx, myConn, time.Duration(2*time.Minute)); !ready {
return fmt.Errorf("myConn: %w, timeout: %s", ErrWaitReadyTimeout, "2m")
}
go func() {
if err := fooUpdater(ctx, dataBuffer, myConn); err != nil {
log.Errorf("foo updater: %v", err)
}
reconnect <- struct{}{}
}()
select {
case <-ctx.Done():
return nil
case <-reconnect:
}
}
}
func fooUpdater(ctx context.Context, dataBuffer custom.CircularBuffer, myConn *grpc.ClientConn) error {
clientStream, err := myConn.Stream(ctx) // custom pb code, returns grpc.ClientConn.NewStream(...)
if err != nil {
return fmt.Errorf("could not obtain stream: %w", err)
}
for {
select {
case <-ctx.Done():
return nil
case data := <-dataBuffer:
if err := clientStream.Send(data); err != nil {
return fmt.Errorf("could not send data: %w", err)
}
}
}
}
func waitUntilReady(ctx context.Context, conn *grpc.ClientConn, maxTimeout time.Duration) bool {
ctx, cancel := context.WithTimeout(ctx, maxTimeout)
defer cancel()
currentState := conn.GetState()
timeoutValid := true
for currentState != connectivity.Ready && timeoutValid {
timeoutValid = conn.WaitForStateChange(ctx, currentState)
currentState = conn.GetState()
// debug print currentState -> prints IDLE
}
return currentState == connectivity.Ready
}
Debugging hints also welcome :)
Based on the provided code and information, there might be an issue with how ctx.Done is being utilized.
The ctx.Done() is being used in fooUpdater and goroutineOnefunctions. When connection breaks, I believe that the ctx.Done() gets called in both functions, with the following execution order:
Connection breaks, the ctx.Done case in the fooUpdater function gets called, exiting the function. The select statement in the goroutineOne function also executes the ctx.Done case, which exists the function, and the client doesn't reconnect.
Try debugging it to check if both select case blocks get executed, but I believe that is the issue here.
According to the GRPC documentation, the connection is re-established if there is a transient failure otherwise it fails immediately. You can try to verify that the failure is transient by printing the connectivity state.
You should print the error code also to understand Why RPC failed.
Maybe what you have tried is not considered a transient failure.
Also, according to the following entry retry logic does not work with streams: grpc-java: Proper handling of retry on client for service streaming call
Here are the links to the corresponding docs:
https://grpc.github.io/grpc/core/md_doc_connectivity-semantics-and-api.html
https://pkg.go.dev/google.golang.org/grpc#section-readme
Also, check the following entry:
Ways to wait if server is not available in gRPC from client side

How to acknowledge if the message is sent successfully from the client side in a client side streaming rpc (GRPC)

I am having a client side rpc which sends messages to the server. The messages should be reliable and should not be lost. I am using redis pub/sub to publish the message and subscribe from it and send it using the client streaming rpc. I need to establish a protocol with acknowledgement where every message sent from the client should be acknowledged by the server and the client should retry the message if the message is not recieved by the server.
My Current Implementation
I converted the client stream to Bi-directional stream and send an acknowledgement message back to the client.
The client after sending a message, check for acknowledgement and then continue with the next message.
This is working fine, but the down side is. I get a delay of 0.5 to 1 sec for a message. This is too much, because the message is produced at a faster rate.
What I need is a delay of 0.1 second. Kindly help me with this. Or any other suggestion is appreciated.
Thank you
func (h *Handler) SendCommandResponsesFromRedis(ctx context.Context) {
sub := h.RedisClient.Subscribe("command_responses")
var rccStream rcc.Agent_SendCommandResponseStreamClient
var isFailed = false
var ackChan = make(chan bool)
// var err error
for msg := range sub.Channel() {
response := &agent.SendCommandResponseRequest{}
err := protojson.Unmarshal([]byte(msg.Payload), response)
if err != nil {
h.Logger.Error("Error unmarshalling data from redis: ", err)
}
for {
// Creating stream
if rccStream == nil || isFailed {
fmt.Println("Creating stream")
if rccStream != nil {
rccStream.Context().Done()
}
rccStream, err = h.Client.SendCommandResponseStream(ctx)
go func() {
for {
select {
case <-ctx.Done():
return
default:
ack, err := rccStream.Recv()
if err != nil {
h.Logger.Error("Error receiving command response stream: ", err)
isFailed = true
return
}
if ack.Status {
ackChan <- true
} else {
ackChan <- false
}
}
}
}()
if err != nil {
h.Logger.Error("Error requesting command response stream: ", err)
isFailed = true
time.Sleep(time.Second * 5)
fmt.Println("Retrying...")
continue
}
isFailed = false
}
// Send message in stream
fmt.Println("Sending message in stream")
err = rccStream.Send(&rcc.SendCommandResponseRequest{
Response: response.Response,
})
// Wait for response, if status is not true wait for 5 seconds and try again by continueing the loop
if err != nil {
h.Logger.Error("Error sending command response stream: ", err)
isFailed = true
continue
}
if ack := <-ackChan; ack {
break
} else {
time.Sleep(time.Second * 5)
continue
}
}
}
defer sub.Close()
}

Connect kafka in go(sarama), the consumer can not get message through topic

I just want to follow a demo to try use kafka in go. I can successfully produce message by sarama, but when i want to consume the message, can not get it.
package main
import (
"fmt"
"github.com/Shopify/sarama"
)
// kafka consumer
func main() {
consumer, err := sarama.NewConsumer([]string{"127.0.0.1:9092"}, nil)
if err != nil {
fmt.Printf("fail to start consumer, err:%v\n", err)
return
}
partitionList, err := consumer.Partitions("test")
if err != nil {
fmt.Printf("fail to get list of partition:err%v\n", err)
return
}
fmt.Println(partitionList)
for partition := range partitionList {
pc, err := consumer.ConsumePartition("test", int32(partition), sarama.OffsetNewest)
if err != nil {
fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err)
return
}
defer pc.AsyncClose()
go func(sarama.PartitionConsumer) {
for msg := range pc.Messages() {
fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v", msg.Partition, msg.Offset, msg.Key, msg.Value)
}
}(pc)
}
}
The return of the code is
[0]
-1
But actually i can get the message through kafka-console-consumer.
I believe you are not wating for the messages to come...
here is the list fo issues you have in your code:
defer pc.AsyncClose() will trigger on function exit, not scope exit.
goroutine is launching into nowhere... nothing blocks or wait for the results to come.
go func(sarama.PartitionConsumer) {
for msg := range pc.Messages() {
fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v", msg.Partition, msg.Offset, msg.Key, msg.Value)
}
}(pc)
not passing argument to goroutine. go func(sarama.PartitionConsumer) { this is only type. go func(pc sarama.PartitionConsumer) {.
Remove goroutine, and just check the consumer channel if you want to make hello world example.

gobwas/ws Send opPing over net.Conn

Can somebody help me understand what am I doing wrong here, all I'm trying to do is write a Ping message over a net.Conn instance (server), and reply back with a Pong which is expected on a net.Conn instance (client).
I have annotated the code with some errors that I receive.
reader.go
func read(conn net.Conn) {
for {
conn.SetReadDeadline(time.Now().Add(2 * time.Second))
_, op, err := wsutil.ReadClientData(conn)
if err != nil {
log.Printf("wsmanager read: %v", err) // read: write pipe: deadline exceeded
return
}
if op != ws.OpPing {
continue
}
c.conn.SetWriteDeadline(time.Now().Add(3 * time.Second))
if err = wsutil.WriteServerMessage(c.conn, ws.OpPong, []byte{}); err != nil {
log.Printf("wsmanager: send pong error: %v", err)
return
}
}
}
// reader_test.go
client, server := net.Pipe()
go read(server) // starts the loop above
err := wsutil.WriteClientMessage(server, ws.OpPing, []byte{})
if err != nil {
t.Fatalf("failed sending pings message %v", err)
}
_, op, err := wsutil.ReadServerData(client)
if err != nil {
t.Errorf("exp no err, got %v", err)
}
if op != ws.OpPong {
t.Errorf("exp ws.OpPong, got %v", op)
}
Thank you for using this library :)
As the doc states, the ReadData functions read data from the connection; that is, application specific data, not the control messages. Control frames are handled implicitly in these functions. If you want to read any kind of message, you could use wsutil.Reader or the plain ws.Read functions.
https://godoc.org/github.com/gobwas/ws/wsutil#ReadClientData

How to test TCP disconnection in GO

I do have a process receiving data from second local process. I need to test if connection errors are handled well AND if it automatically reconnects and keep receiving data after a disconnection.
To do so I am trying to make it disconnect abruptly or put the TCP connection in an error state from a unit test.
As seen in this question to check if a connection is closed I am checking for data to come and test if it returns an error.
I am not sure how to:
close the connection ungracefully
make it be in an error state
This is the essence of my data receiver:
import (
"bufio"
"encoding/json"
"fmt"
"io"
"net"
)
type Message struct {
ID string `json:"id"`
}
func ReceiveData(listener Listener) {
var tcpConn net.Conn
var addr string = "127.0.0.1:9999"
tcpConn, err := net.Dial("tcp", addr)
socketReader := bufio.NewReader(tcpConn)
decoder := json.NewDecoder(socketReader)
for {
var msg Message
if err := decoder.Decode(&msg); err == io.EOF {
listener.ProcessUpdate(Message{}, fmt.Errorf("Received EOF"), nil)
tcpConn = nil
return
} else if err != nil {
listener.ProcessUpdate(Message{}, nil, fmt.Errorf("Error decoding message: %s", err.Error()))
tcpConn = nil
return
}
// process message
_ = msg
// Test disconnection
// This does not disconnect:
// tcpConn = nil
// This does but gracefully:
// tcpConn.Close()
}
I am not mocking the TCP connection as I'd like to try with the real data producer. If is needed I'll look at it.
A solution is to set a deadline to the TCP connection itself:
tcpConn.SetDeadline(time.Now())
Later this will trigger a timeout error which can be caught with:
err := decoder.Decode(&msg);
if err != nil {
if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
fmt.Errorf("TCP timeout : %s", err.Error())
} else {
fmt.Errorf("Received error decoding message: %s", err.Error())
}
}

Resources