XACK is not deleting the message, even if it is processed successfully? - go

I am trying to implement redis stream where we have a producer.
package producer
import (
"RedisStream/models"
"encoding/json"
"fmt"
"github.com/garyburd/redigo/redis"
)
type Producer struct {
streamName string
}
func NewProducer(streamName string) *Producer {
return &Producer{streamName: streamName}
}
func (p *Producer) WriteEvents(conn redis.Conn, key string) {
// Create a new struct
employee := models.Employee{
Name: "ashutosh",
Employer: "self-employee",
}
// Convert struct to JSON
e, _ := json.Marshal(employee)
// Send key and value to Redis stream
_, err := conn.Do("XADD", p.streamName, "*", key, e)
if err != nil {
fmt.Println(err)
}
fmt.Println("Successfully sent data to Redis stream")
}
then I have implemented a consumer
func (c *Consumer) ReadEventsCons1() {
// Connect to Redis
conn, err := redis.Dial("tcp", ":6379")
if err != nil {
fmt.Println(err)
return
}
defer conn.Close()
for {
// Read key and value from Redis stream
reply, err := conn.Do("XREADGROUP", "GROUP", c.groupName[0], "ashu", "COUNT", "1", "STREAMS", c.streamName, ">")
vs, err := redis.Values(reply, err)
if err != nil {
if errors.Is(err, redis.ErrNil) {
continue
}
fmt.Printf("Error: %+v", err)
}
// Get the first and only value in the array since we're only
// reading from one stream "some-stream-name" here.
vs, err = redis.Values(vs[0], nil)
if err != nil {
fmt.Printf("Error: %+v", err)
}
// Ignore the stream name as the first value as we already have
// that in hand! Just get the second value which is guaranteed to
// exist per the docs, and parse it as some stream entries.
res, err := entries(vs[1], nil)
if err != nil {
fmt.Errorf("error parsing entries: %w", err)
}
for _, val := range res {
for k, v := range val.Fields {
empl := &models.Employee{}
_ = json.Unmarshal(v, empl)
fmt.Printf("From Consumer Ashu: Key: %s and val: %+v \n", k, empl)
}
reply, err := redis.Int(conn.Do("XACK", c.streamName, c.groupName[0], val.ID))
if reply != 1 {
fmt.Printf("failed to ack: err: %+v", err)
}
}
}
}
Once a consumer from a consumergroup successfully processed a message, I sent acknowledgement to redis.But messages still resides in redis stream. because post running
XLEN streamName
I can see length is growing. This may create memory challenge, since messages are residing in perpetuity. Is there any intelligent way to handle this issue?

Related

kafka retry many times when i download large file

I am newbie in kafka, i try build a service send mail with attach files.
Execution flow:
Kafka will receive a message to send mail
function get file will download file from url , scale image, and save file
when send mail i will get files from folder and attach to form
Issues:
when i send mail with large files many times , kafka retry many times, i will receive many mail
kafka error: "kafka server: The provided member is not known in the current generation"
I listened MaxProcessingTime , but i try to test a mail with large file, it still work fine
Kafka info : 1 broker , 3 consumer
func (s *customerMailService) SendPODMail() error { filePaths, err := DownloadFiles(podURLs, orderInfo.OrderCode)
if err != nil{
countRetry := 0
for countRetry <= NUM_OF_RETRY{
filePaths, err = DownloadFiles(podURLs, orderInfo.OrderCode)
if err == nil{
break
}
countRetry++
}
}
err = s.sendMailService.Send(ctx, orderInfo.CustomerEmail, tmsPod, content,filePaths)}
function download file :
func DownloadFiles(files []string, orderCode string) ([]string, error) {
var filePaths []string
err := os.Mkdir(tempDir, 0750)
if err != nil && !os.IsExist(err) {
return nil, err
}
tempDirPath := tempDir + "/" + orderCode
err = os.Mkdir(tempDirPath, 0750)
if err != nil && !os.IsExist(err) {
return nil, err
}
for _, fileUrl := range files {
fileUrlParsed, err := url.ParseRequestURI(fileUrl)
if err != nil {
logrus.WithError(err).Infof("Pod url is invalid %s", orderCode)
return nil, err
}
extFile := filepath.Ext(fileUrlParsed.Path)
dir, err := os.MkdirTemp(tempDirPath, "tempDir")
if err != nil {
return nil, err
}
f, err := os.CreateTemp(dir, "tmpfile-*"+extFile)
if err != nil {
return nil, err
}
defer f.Close()
response, err := http.Get(fileUrl)
if err != nil {
return nil, err
}
defer response.Body.Close()
contentTypes := response.Header["Content-Type"]
isTypeAllow := false
for _, contentType := range contentTypes {
if contentType == "image/png" || contentType == "image/jpeg" {
isTypeAllow = true
}
}
if !isTypeAllow {
logrus.WithError(err).Infof("Pod image type is invalid %s", orderCode)
return nil, errors.New("Pod image type is invalid")
}
decodedImg, err := imaging.Decode(response.Body)
if err != nil {
return nil, err
}
resizedImg := imaging.Resize(decodedImg, 1024, 0, imaging.Lanczos)
imaging.Save(resizedImg, f.Name())
filePaths = append(filePaths, f.Name())
}
return filePaths, nil}
function send mail
func (s *tikiMailService) SendFile(ctx context.Context, receiver string, templateCode string, data interface{}, filePaths []string) error {
path := "/v1/emails"
fullPath := fmt.Sprintf("%s%s", s.host, path)
formValue := &bytes.Buffer{}
writer := multipart.NewWriter(formValue)
_ = writer.WriteField("template", templateCode)
_ = writer.WriteField("to", receiver)
if data != nil {
b, err := json.Marshal(data)
if err != nil {
return errors.Wrapf(err, "Cannot marshal mail data to json with object %+v", data)
}
_ = writer.WriteField("params", string(b))
}
for _, filePath := range filePaths {
part, err := writer.CreateFormFile(filePath, filepath.Base(filePath))
if err != nil {
return err
}
pipeReader, pipeWriter := io.Pipe()
go func() {
defer pipeWriter.Close()
file, err := os.Open(filePath)
if err != nil {
return
}
defer file.Close()
io.Copy(pipeWriter, file)
}()
io.Copy(part, pipeReader)
}
err := writer.Close()
if err != nil {
return err
}
request, err := http.NewRequest("POST", fullPath, formValue)
if err != nil {
return err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := s.doer.Do(request)
if err != nil {
return errors.Wrap(err, "Cannot send request to send email")
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New(fmt.Sprintf("Send email with code %s error: status code %d, response %s",
templateCode, resp.StatusCode, string(b)))
} else {
logrus.Infof("Send email with attachment ,code %s success with response %s , box-code", templateCode, string(b),filePaths)
}
return nil
}
Thank
My team found my problem when I redeploy k8s pods, which lead to conflict leader partition causing rebalance. It will try to process the remaining messages in buffer of pods again.
Solution: I don't fetch many messages saved in buffer , I just get a message and process it by config :
ChannelBufferSize = 0
Example conflict leader parition:
consumer A and B startup in the same time
consumer A registers itself as leader, and owns the topic with all partitions
consumer B registers itself as leader, and then begins to rebalance and owns all partitions
consumer A rebalance and obtains all partitions, but can not consume because the memberId is old and need a new one
consumer B rebalance again and owns the topic with all partitions, but it's already obtained by consumer A
My two cents: in case of very big attachments, the consumer takes quite a lot of time to read the file and to send it as an attachment.
This increases the amount of time between two poll() calls. If that time is greater than max.poll.interval.ms, the consumer is thought to be failed and the partition offset is not committed. As a result, the message is processed again and eventually, if by chance the execution time stays below the poll interval, the offset is committed. The effect is a multiple email send.
Try increasing the max.poll.interval.ms on the consumer side.

How to cache a TCP reverse proxy data transmission?

I've accomplished implementing TCP reverse proxy in GoLang. But unfortunately couldn't come up with implementing caching to a TCP reverse proxy. Is it possible to do so, if yes, is there any resource out there? Is caching possible on a TCP (Transport Layer of Network)?
Here's the simple TCP reverse proxy in Golang.
package main
import (
"io"
"log"
"net"
)
//Proxy struct
type Proxy struct {
laddr, raddr *net.TCPAddr
lconn, rconn io.ReadWriteCloser
errorSignal chan bool
}
// New Create a new Proxy instance.
func New(lconn *net.TCPConn, laddr, raddr *net.TCPAddr) *Proxy {
return &Proxy{
lconn: lconn,
laddr: laddr,
raddr: raddr,
errorSignal: make(chan bool),
}
}
//TCPAddressResolver resolves an address and returns to a struct having ip and port.
func TCPAddressResolver(addr string) (tcpAddress *net.TCPAddr, err error) {
tcpAddress, err = net.ResolveTCPAddr("tcp", addr)
return
}
func main() {
listenerAddress, err := TCPAddressResolver(":8080")
if err != nil {
log.Fatalf("Failed to resolve local address: %v", err)
}
remoteAddress, err := TCPAddressResolver(":3000")
if err != nil {
log.Fatalf("Failed to resolve remote address: %v", err)
}
listener, err := net.ListenTCP("tcp", listenerAddress)
if err != nil {
log.Fatalf("Failed to open local port to listen: %v", err)
}
log.Printf("Simple Proxy started on: %d and forwards to port %d", listenerAddress.Port, remoteAddress.Port)
for {
conn, err := listener.AcceptTCP()
if err != nil {
log.Fatalf("Failed to accept connection: %v", err)
continue
}
var p *Proxy
// HTTP is a stateless protocol thus a proxy needs to reinitiate the new next incoming call (conn)
// each time it finishes handling the previous one.
p = New(conn, listenerAddress, remoteAddress)
p.Start()
}
}
//Start initiates transmission of data to and from the remote to client side.
func (p *Proxy) Start() {
defer p.lconn.Close()
var err error
p.rconn, err = net.DialTCP("tcp", nil, p.raddr)
if err != nil {
log.Fatalf("Remote connection failure: %v", err)
}
defer p.rconn.Close()
go p.CopySrcDst(p.lconn, p.rconn)
go p.CopySrcDst(p.rconn, p.lconn)
//Wait for everything to close -- This one blocks the routine.
<-p.errorSignal
log.Printf("Closing Start routine \n")
}
func (p *Proxy) err(err error) {
if err != io.EOF {
log.Printf("Warning: %v: Setting error signal to true", err)
}
p.errorSignal <- true
}
//CopySrcDst copies data from src to dest
func (p *Proxy) CopySrcDst(src, dst io.ReadWriteCloser) {
buff := make([]byte, 1024)
for {
n, err := src.Read(buff)
if err != nil {
// Reading error.
p.err(err)
return
}
dataFromBuffer := buff[:n]
n, err = dst.Write(dataFromBuffer)
if err != nil {
// Writing error.
p.err(err)
return
}
}
}
You are asking how to save data read from an io.Reader. That's different from caching.
The easiest approach is to tee the reader into a buffer.
While you are at it, you might as well use io.Copy instead of the similar code in the question. The code in the question does not handle the case when read returns n > 0 and a non-nil error.
Use an error group to coordinate waiting for the goroutines and collecting error status.
var g errgroup.Group
var rbuf, lbuf bytes.Buffer
g.Go(func() error {
_, err := io.Copy(lconn, io.TeeReader(p.rconn, &rbuf))
return err
})
g.Go(func() error {
_, err := io.Copy(rconn, io.TeeReader(p.lconn, &lbuf))
return err
})
if err := g.Wait(); err != nil {
// handle error
}
// rbuf and lbuf have the contents of the two streams.
The name of the programming language is "Go", not "Golang" or "GoLang".

Go GRPC Bidirectional Stream Performance

We are developing a high frequency trading platform and in one of our components we have implemented the grpc with golang. And we needed to use bidirectional streaming in one of our usecases , we made a sample implementation as in below code , however when we test the performance of the code by checking the difference between timestamps of the logs in
Recv Time %v Index: %v Num: %v
Send Time %v, Index: %v, Num: %v
we found out that calling .Send method of the stream from client side and receiving the same data by calling .Recv on the server side tooks approximately 400-800 microseconds which is too low for us. We need maximum 10-50 microseconds performance , and when we read the guidelines we saw that grpc can go up to nanoseconds if both client and server is in the same computer (Which is exactly our case)
So I think we are missing some options or some performance tricks about it. Does anyone know what we can do to increase this performance problem
Client Code:
package main
import (
"context"
"log"
"math/rand"
pb "github.com/pahanini/go-grpc-bidirectional-streaming-example/src/proto"
"time"
"google.golang.org/grpc"
)
func main() {
rand.Seed(time.Now().Unix())
// dail server
conn, err := grpc.Dial(":50005", grpc.WithInsecure())
if err != nil {
log.Fatalf("can not connect with server %v", err)
}
// create stream
client := pb.NewMathClient(conn)
stream, err := client.Max(context.Background())
if err != nil {
log.Fatalf("openn stream error %v", err)
}
var max int32
ctx := stream.Context()
done := make(chan bool)
msgCount := 100
fromMsg := 0
// first goroutine sends random increasing numbers to stream
// and closes int after 10 iterations
go func() {
for i := 1; i <= msgCount; i++ {
// generate random nummber and send it to stream
rnd := int32(i)
req := pb.Request{Num: rnd}
if i-1 >= fromMsg {
sendTime := time.Now().UnixNano()
log.Printf("Send Time %v, Index: %v, Num: %v", sendTime,i-1,req.Num)
}
if err := stream.Send(&req); err != nil {
log.Fatalf("can not send %v", err)
}
//afterSendTime := time.Now().UnixNano()
//log.Printf("After Send Time %v", afterSendTime)
//log.Printf("---------------")
//log.Printf("%d sent", req.Num)
//time.Sleep(time.Millisecond * 200)
}
if err := stream.CloseSend(); err != nil {
log.Println(err)
}
}()
// third goroutine closes done channel
// if context is done
go func() {
<-ctx.Done()
if err := ctx.Err(); err != nil {
log.Println(err)
}
close(done)
}()
<-done
log.Printf("finished with max=%d", max)
}
Server Code:
package main
import (
"io"
"log"
"net"
"time"
pb "github.com/pahanini/go-grpc-bidirectional-streaming-example/src/proto"
"google.golang.org/grpc"
)
type server struct{}
func (s server) Max(srv pb.Math_MaxServer) error {
log.Println("start new server")
var max int32
ctx := srv.Context()
i := 0
fromMsg := 0
for {
// exit if context is done
// or continue
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// receive data from stream
req, err := srv.Recv()
if err == io.EOF {
// return will close stream from server side
log.Println("exit")
return nil
}
if err != nil {
log.Printf("receive error %v", err)
continue
}
if i >= fromMsg {
recvTime := time.Now().UnixNano()
log.Printf("Recv Time %v Index: %v Num: %v", recvTime,i,req.Num)
}
i++
// continue if number reveived from stream
// less than max
if req.Num <= max {
continue
}
// update max and send it to stream
/*
max = req.Num
resp := pb.Response{Result: max}
if err := srv.Send(&resp); err != nil {
log.Printf("send error %v", err)
}
*/
//log.Printf("send new max=%d", max)
}
}
func main() {
// create listiner
lis, err := net.Listen("tcp", ":50005")
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
// create grpc server
s := grpc.NewServer()
pb.RegisterMathServer(s, server{})
// and start...
if err := s.Serve(lis); err != nil {
log.Fatalf("failed to serve: %v", err)
}
}

How do I send Protobuf Messages via a Kafka Producer

I am using the Sarama Library to send messages through a Producer.
This allows me to send strings. My goal is to send Protobuf Messages
msg := &sarama.ProducerMessage{
Topic: *topic,
Value: sarama.StringEncoder(content),
}
This is a sample proto class that I have
message Pixel {
// Session identifier stuff
int64 timestamp = 1; // Milliseconds from the epoch
string session_id = 2; // Unique Identifier... for parent level0top
string client_name = 3; // Client-name/I-key
string ip = 10;
repeated string ip_list = 11;
string datacenter = 12;
string proxy_type = 13;
Please can you provide me an example of how I can send protobuf messages.
You need to use proto#Marshal and sarama#ByteEncoder on producer side and proto#Unmarshal on consumer side.
Producer:
pixelToSend := &pixel.Pixel{SessionId: t.String()}
pixelToSendBytes, err := proto.Marshal(pixelToSend)
if err != nil {
log.Fatalln("Failed to marshal pixel:", err)
}
msg := &sarama.ProducerMessage{
Topic: topic,
Value: sarama.ByteEncoder(pixelToSendBytes),
}
Consumer:
receivedPixel := &pixel.Pixel{}
err := proto.Unmarshal(msg.Value, receivedPixel)
if err != nil {
log.Fatalln("Failed to unmarshal pixel:", err)
}
log.Printf("Pixel received: %s", receivedPixel)
Complete example:
package main
import (
pixel "example/pixel"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/Shopify/sarama"
"github.com/golang/protobuf/proto"
)
func main() {
topic := "your-topic-name"
brokerList := []string{"localhost:29092"}
producer, err := newSyncProducer(brokerList)
if err != nil {
log.Fatalln("Failed to start Sarama producer:", err)
}
go func() {
ticker := time.NewTicker(time.Second)
for {
select {
case t := <-ticker.C:
pixelToSend := &pixel.Pixel{SessionId: t.String()}
pixelToSendBytes, err := proto.Marshal(pixelToSend)
if err != nil {
log.Fatalln("Failed to marshal pixel:", err)
}
msg := &sarama.ProducerMessage{
Topic: topic,
Value: sarama.ByteEncoder(pixelToSendBytes),
}
producer.SendMessage(msg)
log.Printf("Pixel sent: %s", pixelToSend)
}
}
}()
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
partitionConsumer, err := newPartitionConsumer(brokerList, topic)
if err != nil {
log.Fatalln("Failed to create Sarama partition consumer:", err)
}
log.Println("Waiting for messages...")
for {
select {
case msg := <-partitionConsumer.Messages():
receivedPixel := &pixel.Pixel{}
err := proto.Unmarshal(msg.Value, receivedPixel)
if err != nil {
log.Fatalln("Failed to unmarshal pixel:", err)
}
log.Printf("Pixel received: %s", receivedPixel)
case <-signals:
log.Print("Received termination signal. Exiting.")
return
}
}
}
func newSyncProducer(brokerList []string) (sarama.SyncProducer, error) {
config := sarama.NewConfig()
config.Producer.Return.Successes = true
// TODO configure producer
producer, err := sarama.NewSyncProducer(brokerList, config)
if err != nil {
return nil, err
}
return producer, nil
}
func newPartitionConsumer(brokerList []string, topic string) (sarama.PartitionConsumer, error) {
conf := sarama.NewConfig()
// TODO configure consumer
consumer, err := sarama.NewConsumer(brokerList, conf)
if err != nil {
return nil, err
}
partitionConsumer, err := consumer.ConsumePartition(topic, 0, sarama.OffsetOldest)
if err != nil {
return nil, err
}
return partitionConsumer, err
}

binary-to-text encoding that replace a byte into a predefined unique string in golang

I am trying to build a binary-to-text encoder with the ability to replace every byte into a predefined unique string and then use the same string set to decode back into binary.
I am able to make the encoder and decoder for simple .txt files but I want to make this workable for .zip files too.
Encoder :
package main
import (
"archive/zip"
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
)
func main() {
// keys.json is a 256 words dictionary for every byte
keysFile, err := os.Open("keys.json")
if err != nil {
log.Printf("unable to read keys.json file , error : %v", err)
return
}
var keys []string
byteValue, _ := ioutil.ReadAll(keysFile)
if err := json.Unmarshal(byteValue, &keys); err != nil {
log.Printf("unable to unmarshal array , error : %v", err)
return
}
Encoder(keys)
}
func listFiles(file *zip.File) ([]byte, error) {
fileToRead, err := file.Open()
if err != nil {
msg := "Failed to open zip %s for reading: %s"
return nil, fmt.Errorf(msg, file.Name, err)
}
b := make([]byte, file.FileInfo().Size())
fileToRead.Read(b)
defer fileToRead.Close()
if err != nil {
return nil, fmt.Errorf("failed to read zip file : %s for reading , error : %s", file.Name, err)
}
return b, nil
}
func Encoder(keys []string) {
read, err := zip.OpenReader("some.zip")
if err != nil {
msg := "Failed to open: %s"
log.Fatalf(msg, err)
}
defer read.Close()
var encodedBytes []byte
f, err := os.Create("result.txt")
w := bufio.NewWriter(f)
defer f.Close()
for _, file := range read.File {
readBytes, err := listFiles(file)
if err != nil {
log.Fatalf("Failed to read %s from zip: %s", file.Name, err)
continue
}
for i, b := range readBytes {
for _, eb := range []byte(keys[b] + " ") {
encodedBytes = append(encodedBytes, eb)
}
}
}
_, err = w.Write(encodedBytes)
if err != nil {
log.Printf("error :%v", err)
return
}
}
Decoder :
func Decoder(keys []string) {
inputFile, err := os.Open("result.txt")
if err != nil {
log.Printf("unable to read file , error : %v", err)
return
}
inputBytes, _ := ioutil.ReadAll(inputFile)
var (
current []byte
decoded []byte
)
for _, c := range inputBytes {
if c != 32 {
current = append(current, c)
} else {
for i, key := range keys {
if string(current) == key {
decoded = append(decoded, byte(i))
break
}
}
current = []byte{}
}
}
// here i want the decoded back into zip file
}
here is a similar one in nodejs.
Two things:
You are dealing with spaces correctly, but not with newlines.
Your decoder loop is wrong. As far as I can tell, it should look like the following:
for i, key := range keys {
if string(current)==key {
decoded=append(decoded,i)
break
}
}
Also, your decoded is an int-array, not a byte-array.

Resources