Could one connection support multiple channels in go api for rabbitmq? - go

package main
import (
"fmt"
"github.com/streadway/amqp"
"time"
)
// Every connection should declare the topology they expect
func setup(url, queue string) (*amqp.Connection, *amqp.Channel, error) {
//setup connection
conn, err := amqp.Dial(url)
if err != nil {
return nil, nil, err
}
//build channel in the connection
ch, err := conn.Channel()
if err != nil {
return nil, nil, err
}
//queue declare
if _, err := ch.QueueDeclare(queue, false, true, false, false, nil); err != nil {
return nil, nil, err
}
return conn, ch, nil
}
func main() {
//amqp url
url := "amqp://guest:guest#127.0.0.1:5672";
for i := 1; i <= 2; i++ {
fmt.Println("connect ", i)
//two goroutine
go func() {
//queue name
queue := fmt.Sprintf("example.reconnect.%d", i)
//setup channel in the tcp connection
_, pub, err := setup(url, queue)
if err != nil {
fmt.Println("err publisher setup:", err)
return
}
// Purge the queue from the publisher side to establish initial state
if _, err := pub.QueuePurge(queue, false); err != nil {
fmt.Println("err purge:", err)
return
}
//publish msg
if err := pub.Publish("", queue, false, false, amqp.Publishing{
Body: []byte(fmt.Sprintf("%d", i)),
}); err != nil {
fmt.Println("err publish:", err)
return
}
//keep running
for{
time.Sleep(time.Second * 20)
}
}()
}
//keep running
for {
time.Sleep(time.Second * 20)
}
}
I thought there is only one connection between the program and mq-server,
but there are two connection,one connection can only support one channel,why?
can't the two goroutine share the same tcp connection?
Socket descriptor can share in all threads of a process in the theory.
Why the two goroutine don't share one socket but have their own channel?
The model by hand:
The real model in rabbitmq:

Looking at the source for the library it appears as though you can call conn.Channel() as many times as you like and it creates a new stream of communication over the same connection.
Ok, I tried it, here's a working example... One goroutine, one connection, two channels
I setup the receiver, then send a message, then read from the receiver channel
if you wanted multiple queue's bound in one goroutine, you would call rec.Consume twice and then select across the queues.
package main
import (
"fmt"
"github.com/streadway/amqp"
"os"
)
func main() {
conn, err := amqp.Dial("amqp://localhost")
e(err)
defer conn.Close()
fmt.Println("Connected")
rec, err := conn.Channel()
e(err)
fmt.Println("Setup receiver")
rq, err := rec.QueueDeclare("go-test", false, false, false, false, nil)
e(err)
msgs, err := rec.Consume(rq.Name, "", true, false, false, false, nil)
e(err)
fmt.Println("Setup sender")
send, err := conn.Channel()
e(err)
sq, err := send.QueueDeclare("go-test", false, false, false, false, nil)
e(err)
fmt.Println("Send message")
err = send.Publish("", sq.Name, false, false, amqp.Publishing{
ContentType: "text/plain",
Body: []byte("This is a test"),
})
e(err)
msg := <-msgs
fmt.Println("Received from:", rq, "msg:", string(msg.Body))
}
func e(err error) {
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
Output on my box:
$ go run rmq.go
Connected
Setup receiver
Setup sender
Send message
Received from: {go-test 0 0} msg: This is a test

Related

Consuming from multiple queue

I have a rabbit mq instance that I use to listen to changes. However, I have different messages that I need to consume. What I am not sure about now is how to go about it. Would I need to just keep duplicating my Consume method and what is the danger of doing that if any.
type Consumer struct {
conn *amqp.Connection
db *mongo.Database
queueName string
}
func (consumer *Consumer) setup() error {
_, err := consumer.conn.Channel()
if err != nil {
return err
}
return nil
}
// NewConsumer returns a new Consumer
func NewConsumer(conn *amqp.Connection, db *mongo.Database) (Consumer, error) {
consumer := Consumer{
conn: conn,
db: db,
}
return consumer, nil
}
// Listen will listen for all new Queue publications
// and print them to the console.
func (consumer *Consumer) Listen(topics []string) error {
ch, err := consumer.conn.Channel()
if err != nil {
return err
}
defer ch.Close()
msgs, err := ch.Consume("charge.get", "", true, false, false, false, nil)
if err != nil {
return err
}
forever := make(chan bool)
go func() {
for msg := range msgs {
switch msg.RoutingKey {
case "charge.get":
worker.Refund(paymentRepo.NewPaymentRepository(consumer.db), msg.Body)
}
// acknowledege received event
log.Printf("Received a message: %s ROUTIG KEY %s", msg.Body, msg.RoutingKey)
}
}()
log.Printf("[*] Waiting for message [Exchange, Queue][%s, %s]. To exit press CTRL+C", " getExchangeName()", "q.Name")
<-forever
return nil
}
Do I just add another msgs, err := ch.Consume("charge.update", "", true, false, false, false, nil) below the first one to consume from this queue? Or how can I go about this.

golang rabbitmq channel.consume SIGSEGV

Folks,
Am trying to write a wrapper library for the rabbitmq implementation. For some strange reason, I am not able to consume from an existing queue.
msgs, err := w.AMQP.Channel.Consume( is causing:
2019-10-14T13:58:56.462-0400 info worker/worker.go:27 [Initializing NewWorker]
2019-10-14T13:58:56.462-0400 debug worker/worker.go:43 Starting Worker
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x14d024e]
The docs example I followed is simple, and the interface to the rabbitmq driver is simple. I dont see why im getting a invalid memory address error.
My implementation is identical to how the article lays it out, for reference, here is all the code i have:
// StartConsumingTlcFHVDrivers subscribes to queue and starts to do it's job
func (w *Worker) StartConsuming() {
queueName := w.options.Rabbit.AMQP.QueueName
w.logger.Debugf("Starting %s Worker", queueName)
msgs, err := w.AMQP.Channel.Consume(
queueName,
"",
false,
false,
false,
false,
nil,
)
if err != nil {
logger.Fatalf("Could not register consumer, err:%s", err)
}
forever := make(chan bool)
go func() {
for d := range msgs {
w.logger.Infof("Received a message: %s", d.Body)
}
}()
w.logger.Infof("Waiting for messages on %s", queueName)
<-forever
}
Before I finish the wrapper with my own interface, im setting up the client this way:
import ""github.com/streadway/amqp""
type Client struct {
options Options
logger ilogger.Logger
metrics imetrics.Client
Channel amqp.Channel
}
func NewRabbitClient(logger ilogger.Logger, metrics imetrics.Client, options Options) *Client {
var err error
conn, err := amqp.Dial(options.AMQPConnectionURL)
if err != nil {
logger.Fatalf("%s: %s", "Can't connect to AMQP", err)
}
defer conn.Close()
// create a dedicated channel per queue
channel, err := conn.Channel()
if err != nil {
logger.Fatalf("%s: %s", "Can't create a amqpChannel", err)
}
defer channel.Close()
// declare an Exchange
err = channel.ExchangeDeclare(options.ExchangeName, options.ExchangeType, true, false, false, false, nil)
// declare a Queue
_, err = channel.QueueDeclare(options.QueueName, true, false, false, false, nil)
if err != nil {
logger.Fatalf("Could not declare %s queue, err:%s", options.QueueName, err)
}
// bind a Queue to the Exchange
err = channel.QueueBind(options.QueueName, "", options.ExchangeName, false, nil)
// Qos on the Channel
err = channel.Qos(1, 0, false)
if err != nil {
logger.Fatalf("Could not configure QoS, err:%s", err)
}
return &Client{
metrics: metrics,
logger: logger,
options: options,
Channel: *channel,
}
}
The error is now obvious to me. In the constructor function, do not close the connection/channel. i.e.
conn, err := amqp.Dial(options.AMQPConnectionURL)
if err != nil {
logger.Fatalf("%s: %s", "Can't connect to AMQP", err)
}
instead of
conn, err := amqp.Dial(options.AMQPConnectionURL)
if err != nil {
logger.Fatalf("%s: %s", "Can't connect to AMQP", err)
}
defer conn.Close()
```

How to return a channel

I'm creating a worker to consume messages from a RabitMQ queue. To achieve that, I created the following file named queue.go
package ExternalServices
import (
"../domain"
"encoding/json"
"github.com/streadway/amqp"
"os"
)
const (
catalogQueue = "catalog-queue"
)
func EnqueueMessageCatalog(catalog *domain.Catalog) error {
marshal, err := json.Marshal(*catalog)
if err != nil {
return err
}
jsonVal := string(marshal)
err = enqueue(catalogQueue, jsonVal)
return err
}
func DequeueMessageCatalog() ([]domain.Catalog, error) {
msgs, err := dequeue(catalogQueue)
if err != nil {
return nil, err
}
allCatalogs := make([]domain.Catalog, len(msgs))
for _, currMsg := range msgs {
var currCatalog domain.Catalog
err = json.Unmarshal([]byte(currMsg), &currCatalog)
if err != nil {
return nil, err
}
}
return allCatalogs, nil
}
func openConnection() (*amqp.Connection, *amqp.Channel, error) {
conn, err := amqp.Dial(os.Getenv("RabbitMQConStr"))
if err != nil {
return nil, nil, err
}
ch, err := conn.Channel()
if err != nil {
conn.Close()
return nil, nil, err
}
return conn, ch, nil
}
func ensureQueueExists(queueName string, ch *amqp.Channel) (amqp.Queue, error) {
q, err := ch.QueueDeclare(
queueName, // name
false, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
return q, err
}
func enqueue(queueName string, message string) error {
con, ch, err := openConnection()
if err != nil {
return err
}
defer con.Close()
defer ch.Close()
q, err := ensureQueueExists(queueName, ch)
if err != nil {
return err
}
err = ch.Publish(
"", // exchange
q.Name, // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "application/json",
Body: []byte(message),
})
return err
}
func dequeue(queueName string) ([]string, error) {
con, ch, err := openConnection()
if err != nil {
return nil, err
}
defer con.Close()
defer ch.Close()
q, err := ensureQueueExists(queueName, ch)
if err != nil {
return nil, err
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return nil, err
}
jsons := make([]string, len(msgs))
i := 0
for currMsg:= range msgs {
jsons[i] = string(currMsg.Body)
i += 1
}
return jsons, nil
}
However, I got a bit confused at the dequeue function. I want my worker to be notified every time a messages arrives at my queue, so I guess the proper way to do so is to create a string chan to my worker, since I don't want to expose the message channel returned by Consume to it.
This is my worker so far.
package worker
import (
"../external-services"
"log"
)
func StartWorker() {
go func() {
messages, err := ExternalServices.DequeueMessageCatalog();
if err != nil {
// todo log
}
for d := range messages {
log.Printf("Received a message: %s", d)
}
}()
}
How can I modify my dequeue function so it returns a string chan?
After modifying this method to return the string chan, do the lines defer con.Close() and defer ch.Close() need to be deleted from this method?
It's my first project in GoLang so anything you think can increase the quality of the code will be much appreciated :-D
maybe something like this:
msgs, err := ch.Consume(...)
/* handle error */
stringCh := make(chan string)
done := make(chan struct{})
go func() {
defer con.Close()
defer ch.Close()
defer close(stringCh)
for {
select {
case currMsg := <-msgs:
stringCh <- string(currMsg.Body)
case <-done:
return
}
}
}()
return stringCh, done
This is only a sketchy example. Basic idea is spawn another goroutine listen to the message chan returned by Consume. Other details like how to graceful shutdown, dequeue interface,... depend on your needs.
After reading #YSTai response, I realised I miss the go routine creation. This is how my code end up.
worker.go
package main
import (
"../domain"
"../externalservices"
"log"
"strings"
"sync"
)
/*
StartWorker initializes a program that will wait for messages enqueued and process them
*/
func StartWorker() {
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
catalogReceived := make(chan domain.Catalog)
defer close(catalogReceived)
for true {
go func() {
externalservices.DequeueMessageCatalog(catalogReceived)
catalog := <-catalogReceived
website := domain.Website{
Name: strings.ToUpper(catalog.Name),
Zip: catalog.Zip}
externalservices.InsertWebSite(&website)
}()
}
}()
log.Printf(" [*] Waiting for messages")
wg.Wait()
}
func main() {
StartWorker()
}
queue.go
package externalservices
import (
"../domain"
"encoding/json"
"github.com/streadway/amqp"
"os"
)
const (
catalogQueue = "catalog-queue"
)
func EnqueueMessageCatalog(catalog *domain.Catalog) error {
marshal, err := json.Marshal(*catalog)
if err != nil {
return err
}
jsonVal := string(marshal)
err = enqueue(catalogQueue, jsonVal)
return err
}
// DequeueMessageCatalog is nice
func DequeueMessageCatalog(messageChannel chan domain.Catalog) {
message := make(chan []byte)
defer close(message)
for true {
go func() {
dequeue(catalogQueue, message)
}()
currCatalog := domain.Catalog{}
json.Unmarshal([]byte(<-message), &currCatalog)
messageChannel <- currCatalog
}
}
func openConnection() (*amqp.Connection, *amqp.Channel, error) {
connString := os.Getenv("RabbitMQConStr")
conn, err := amqp.Dial(connString)
if err != nil {
return nil, nil, err
}
ch, err := conn.Channel()
if err != nil {
conn.Close()
return nil, nil, err
}
return conn, ch, nil
}
func ensureQueueExists(queueName string, ch *amqp.Channel) (amqp.Queue, error) {
q, err := ch.QueueDeclare(
queueName, // name
false, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
return q, err
}
func enqueue(queueName string, message string) error {
con, ch, err := openConnection()
if err != nil {
return err
}
defer con.Close()
defer ch.Close()
q, err := ensureQueueExists(queueName, ch)
if err != nil {
return err
}
err = ch.Publish(
"", // exchange
q.Name, // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "application/json",
Body: []byte(message),
})
return err
}
func dequeue(queueName string, message chan []byte) error {
con, ch, err := openConnection()
if err != nil {
return err
}
defer con.Close()
defer ch.Close()
q, err := ensureQueueExists(queueName, ch)
if err != nil {
return err
}
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
true, // no-wait
nil, // args
)
if err != nil {
return err
}
for currMsg := range msgs {
message <- currMsg.Body
}
return nil
}

Limiting concurrency when processing messages from RabbitMQ

I'm attempting to read URLs from a queue (RabbitMQ) and make a limited number of concurrent HTTP requests i.e. have a pool of 10 workers making concurrent requests to URLs received from the queue (forever).
So far I've implemented a consumer as per the RabbitMQ tutorials:
https://www.rabbitmq.com/tutorials/tutorial-one-go.html
And have tried a number of methods from examples discovered on the web, ending at the example here:
http://jmoiron.net/blog/limiting-concurrency-in-go/
Unfortunately, my current code runs for approximately one minute and then freezes indefinitely. I've tried adding/moving go routines around but I can't seem to get it to work as intended (I'm very new to Go).
Current code:
package main
import (
"fmt"
"log"
"net/http"
"time"
"github.com/Xide/bloom"
"github.com/streadway/amqp"
)
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
var netClient = &http.Client{
Timeout: time.Second * 10,
}
func getRequest(url string) {
//resp, err := http.Get(string(url))
resp, err := netClient.Get(string(url))
if err != nil {
log.Printf("HTTP request error: %s", err)
return
}
fmt.Println("StatusCode:", resp.StatusCode)
fmt.Println(resp.Request.URL)
}
func main() {
bf := bloom.NewDefaultScalable(0.1)
conn, err := amqp.Dial("amqp://127.0.0.1:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
q, err := ch.QueueDeclare(
"urls", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.Qos(
1, // prefetch count
0, // prefetch size
false, //global
)
failOnError(err, "Failed to set Qos")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
forever := make(chan bool)
concurrency := 10
sem := make(chan bool, concurrency)
go func() {
for d := range msgs {
sem <- true
url := string(d.Body)
if bf.Match(url) == false {
bf.Feed(url)
log.Printf("Not seen: %s", d.Body)
go func(url string) {
defer func() { <-sem }()
getRequest(url)
}(url)
} else {
log.Printf("Already seen: %s", d.Body)
}
d.Ack(false)
}
for i := 0; i < cap(sem); i++ {
sem <- true
}
}()
log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
<-forever
}
You're not properly handling your HTTP responses, leading to a growing set of open connections. Try this:
func getRequest(url string) {
resp, err := netClient.Get(string(url))
if err != nil {
log.Printf("HTTP request error: %s", err)
return
}
// Add this bit:
defer func() {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}()
fmt.Println("StatusCode:", resp.StatusCode)
fmt.Println(resp.Request.URL)
}
This, after you finish reading messages from the channel, seems unnecessary and potentially problematic:
for i := 0; i < cap(sem); i++ {
sem <- true
}
Why fill the sem channel after you've read all the messages from the queue? You've added exactly as many messages to the channel as you expect to read from it, so this is pointless at best, and could cause problems if you make the wrong change to the rest of the code.
Unrelated to your issue, but this is redundant:
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
Per the documentation, Fatalf already exits, so the panic will never be called. If you want to log and panic, try log.Panicf, which is designed for that purpose.
You are adding to sem when you get a message, but only removing from sem when you haven't seen a url.
so, once you've "already seen" 10 urls, your app will lock up.
So you need to add <-sem to your else statement that logs "Already seen".
Either way, that's a fairly odd way to do this kind of concurrency.
Here's a version that does this in a more idiomatic way on Play.
Note, in this version, we just spawn 10 goroutines that listen to the rabbit channel.
package main
import (
"fmt"
"log"
"net/http"
"time"
"github.com/Xide/bloom"
"github.com/streadway/amqp"
)
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
}
}
var netClient = &http.Client{
Timeout: time.Second * 10,
}
func getRequest(url string) {
//resp, err := http.Get(string(url))
resp, err := netClient.Get(string(url))
if err != nil {
log.Printf("HTTP request error: %s", err)
return
}
resp.Body.Close()
fmt.Println("StatusCode:", resp.StatusCode)
fmt.Println(resp.Request.URL)
}
func main() {
bf := bloom.NewDefaultScalable(0.1)
conn, err := amqp.Dial("amqp://127.0.0.1:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
q, err := ch.QueueDeclare(
"urls", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.Qos(
1, // prefetch count
0, // prefetch size
false, //global
)
failOnError(err, "Failed to set Qos")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
concurrency := 10
var wg sync.Waitgroup // used to coordinate when they are done, ie: if rabbit conn was closed
for x := 0; x < concurrency; x++ { // spawn 10 goroutines, all reading from the rabbit channel
wg.Add(1)
go func() {
defer wg.Done() // signal that this goroutine is done
for d := range msgs {
url := string(d.Body)
if bf.Match(url) == false {
bf.Feed(url)
log.Printf("Not seen: %s", d.Body)
getRequest(url)
} else {
log.Printf("Already seen: %s", d.Body)
}
d.Ack(false)
}
log.Println("msgs channel closed")
}()
}
log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
wg.Wait() // when all goroutine's exit, the app exits
}

Golang amqp reconnect

I want to test the restart connection to the rabbitmq server.
On wrote small script to test.
http://play.golang.org/p/l3ZWzG0Qqb
But it's not working.
In step 10, I close the channel and connection. And open them again. And re-create chan amqp.Confirmation ( :75) . And continue the cycle.
But after that, from the chan confirms nothing return.
UPD: code here.
package main
import (
"fmt"
"github.com/streadway/amqp"
"log"
"os"
"time"
)
const SERVER = "amqp://user:pass#localhost:5672/"
const EXCHANGE_NAME = "publisher.test.1"
const EXCHANGE_TYPE = "direct"
const ROUTING_KEY = "publisher.test"
var Connection *amqp.Connection
var Channel *amqp.Channel
func setup(url string) (*amqp.Connection, *amqp.Channel, error) {
conn, err := amqp.Dial(url)
if err != nil {
return nil, nil, err
}
ch, err := conn.Channel()
if err != nil {
return nil, nil, err
}
return conn, ch, nil
}
func main() {
url := SERVER
Connection, Channel, err := setup(url)
if err != nil {
fmt.Println("err publisher setup:", err)
return
}
confirms := Channel.NotifyPublish(make(chan amqp.Confirmation, 1))
if err := Channel.Confirm(false); err != nil {
log.Fatalf("confirm.select destination: %s", err)
}
for i := 1; i <= 3000000; i++ {
log.Println(i)
if err != nil {
fmt.Println("err consume:", err)
return
}
if err := Channel.Publish(EXCHANGE_NAME, ROUTING_KEY, false, false, amqp.Publishing{
Body: []byte(fmt.Sprintf("%d", i)),
}); err != nil {
fmt.Println("err publish:", err)
log.Printf("%+v", err)
os.Exit(1)
return
}
// only ack the source delivery when the destination acks the publishing
confirmed := <-confirms
if confirmed.Ack {
log.Printf("confirmed delivery with delivery tag: %d", confirmed.DeliveryTag)
} else {
log.Printf("failed delivery of delivery tag: %d", confirmed.DeliveryTag)
// TODO. Reconnect will be here
}
if i == 10 {
Channel.Close()
Connection.Close()
while := true
for while {
log.Println("while")
time.Sleep(time.Second * 1)
Connection, Channel, err = setup(url)
if err == nil {
while = false
confirms = Channel.NotifyPublish(make(chan amqp.Confirmation, 1))
log.Printf("%+v", confirms)
}
}
}
time.Sleep(time.Millisecond * 300)
}
os.Exit(1)
}
You should put channel in confirm mode. by calling the channel.Confirm() method.
After closing the connection and even after getting new channel on the same connection, you should call Confirm() method again, since the channel is different from the old channel, and the default for all new channel is not to send confirm.

Resources