How do I send Protobuf Messages via a Kafka Producer - go

Question

I am using the Sarama Library to send messages through a Producer.
This allows me to send strings. My goal is to send Protobuf Messages
msg := &sarama.ProducerMessage{
Topic: *topic,
Value: sarama.StringEncoder(content),
}
This is a sample proto class that I have
message Pixel {
// Session identifier stuff
int64 timestamp = 1; // Milliseconds from the epoch
string session_id = 2; // Unique Identifier... for parent level0top
string client_name = 3; // Client-name/I-key
string ip = 10;
repeated string ip_list = 11;
string datacenter = 12;
string proxy_type = 13;
Please can you provide me an example of how I can send protobuf messages.

Answer 1

You need to use proto#Marshal and sarama#ByteEncoder on producer side and proto#Unmarshal on consumer side.
Producer:
pixelToSend := &pixel.Pixel{SessionId: t.String()}
pixelToSendBytes, err := proto.Marshal(pixelToSend)
if err != nil {
log.Fatalln("Failed to marshal pixel:", err)
}
msg := &sarama.ProducerMessage{
Topic: topic,
Value: sarama.ByteEncoder(pixelToSendBytes),
}
Consumer:
receivedPixel := &pixel.Pixel{}
err := proto.Unmarshal(msg.Value, receivedPixel)
if err != nil {
log.Fatalln("Failed to unmarshal pixel:", err)
}
log.Printf("Pixel received: %s", receivedPixel)
Complete example:
package main
import (
pixel "example/pixel"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/Shopify/sarama"
"github.com/golang/protobuf/proto"
)
func main() {
topic := "your-topic-name"
brokerList := []string{"localhost:29092"}
producer, err := newSyncProducer(brokerList)
if err != nil {
log.Fatalln("Failed to start Sarama producer:", err)
}
go func() {
ticker := time.NewTicker(time.Second)
for {
select {
case t := <-ticker.C:
pixelToSend := &pixel.Pixel{SessionId: t.String()}
pixelToSendBytes, err := proto.Marshal(pixelToSend)
if err != nil {
log.Fatalln("Failed to marshal pixel:", err)
}
msg := &sarama.ProducerMessage{
Topic: topic,
Value: sarama.ByteEncoder(pixelToSendBytes),
}
producer.SendMessage(msg)
log.Printf("Pixel sent: %s", pixelToSend)
}
}
}()
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
partitionConsumer, err := newPartitionConsumer(brokerList, topic)
if err != nil {
log.Fatalln("Failed to create Sarama partition consumer:", err)
}
log.Println("Waiting for messages...")
for {
select {
case msg := <-partitionConsumer.Messages():
receivedPixel := &pixel.Pixel{}
err := proto.Unmarshal(msg.Value, receivedPixel)
if err != nil {
log.Fatalln("Failed to unmarshal pixel:", err)
}
log.Printf("Pixel received: %s", receivedPixel)
case <-signals:
log.Print("Received termination signal. Exiting.")
return
}
}
}
func newSyncProducer(brokerList []string) (sarama.SyncProducer, error) {
config := sarama.NewConfig()
config.Producer.Return.Successes = true
// TODO configure producer
producer, err := sarama.NewSyncProducer(brokerList, config)
if err != nil {
return nil, err
}
return producer, nil
}
func newPartitionConsumer(brokerList []string, topic string) (sarama.PartitionConsumer, error) {
conf := sarama.NewConfig()
// TODO configure consumer
consumer, err := sarama.NewConsumer(brokerList, conf)
if err != nil {
return nil, err
}
partitionConsumer, err := consumer.ConsumePartition(topic, 0, sarama.OffsetOldest)
if err != nil {
return nil, err
}
return partitionConsumer, err
}