RabbitMQ pub/sub implementation not working - go

I've converted the RabbitMQ pub/sub tutorial into the below dummy test. Somehow it is not working as expected.
amqpURL is a valid AMQP service (i.e. RabbitMQ) URL. I've tested it with the queue example and it works. Somehow it fails in "exchange"
I'd expect TestDummy to log " [x] Hello World". Somehow it is not happening. Only the sending half is working as expected.
What did I got wrong?
import (
"fmt"
"log"
"testing"
"github.com/streadway/amqp"
)
func TestDummy(t *testing.T) {
done := exchangeReceive()
exchangeSend("Hello World")
<-done
}
func exchangeSend(msg string) {
failOnError := func(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
log.Printf("exchangeSend: connect %s", amqpURL)
conn, err := amqp.Dial(amqpURL)
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
body := []byte(msg)
err = ch.Publish(
"logs", // exchange
"", // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "text/plain",
Body: []byte(body),
})
failOnError(err, "Failed to publish a message")
log.Printf(" [x] Sent %s", body)
}
func exchangeReceive() <-chan bool {
done := make(chan bool)
failOnError := func(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
log.Printf("exchangeReceive: connect %s", amqpURL)
conn, err := amqp.Dial(amqpURL)
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"logs", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
go func() {
for d := range msgs {
log.Printf(" [x] %s", d.Body)
done <- true
}
}()
log.Printf(" [*] Waiting for logs. To exit press CTRL+C")
return done
}

Some silly mistake here. When exchangeRecieve ends, the defer statments are triggered and hence closed the connections. That's why my rewrite fails.
I've changed my code this way and it worked:
import (
"fmt"
"os"
"testing"
"time"
"github.com/streadway/amqp"
)
func TestDummy(t *testing.T) {
amqpURL := os.Getenv("CLOUDAMQP_URL")
t.Logf(" [*] amqpURL: %s", amqpURL)
results1 := exchangeReceive(t, "consumer 1", amqpURL)
results2 := exchangeReceive(t, "consumer 2", amqpURL)
time.Sleep(50 * time.Millisecond)
exchangeSend(t, amqpURL, "Hello World")
if want, have := "Hello World", <-results1; want != have {
t.Errorf("expected %#v, got %#v", want, have)
}
if want, have := "Hello World", <-results2; want != have {
t.Errorf("expected %#v, got %#v", want, have)
}
}
func exchangeReceive(t *testing.T, name, amqpURL string) <-chan string {
out := make(chan string)
failOnError := func(err error, msg string) {
if err != nil {
t.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
conn, err := amqp.Dial(amqpURL)
failOnError(err, "Failed to connect to RabbitMQ")
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"logs", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
go func() {
for d := range msgs {
t.Logf(" [x] %s received: %s", name, d.Body)
out <- string(d.Body)
}
}()
t.Logf(" [*] %s ready to receive", name)
return out
}
func exchangeSend(t *testing.T, amqpURL, msg string) {
failOnError := func(err error, msg string) {
if err != nil {
t.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
conn, err := amqp.Dial(amqpURL)
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
body := []byte(msg)
err = ch.Publish(
"logs", // exchange
"", // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "text/plain",
Body: []byte(body),
})
failOnError(err, "Failed to publish a message")
t.Logf(" [x] Sent %s", body)
}

Related

Golang RabbitMQ

I followed the tutorial examples, but it reads all message from queue once, how can I read only one message from a queue? Appreciate!
messages, err := channelRabbitMQ.Consume(
"QueueService1", // queue name
"", // consumer
true, // auto-ack
false, // exclusive
false, // no local
false, // no wait
nil, // arguments
)
if err != nil {
log.Println(err)
}
msg := <-messages
fmt.Println(string(msg.Body))
You can use Qos to configure channel to only receive 1 message at a time
url := "..."
queue := "..."
conn, err := amqp.Dial(url)
if err != nil {
log.Fatal("Cannot connect to rabbitmq")
}
ch, err := conn.Channel()
if err != nil {
log.Fatal("Cannot create channel")
}
if _, err := ch.QueueDeclare(queue, false, true, false, false, nil); err != nil {
log.Fatal("Cannot create queue")
}
// Indicate we only want 1 message to acknowledge at a time.
if err := ch.Qos(1, 0, false); err != nil {
log.Fatal("Qos Setting was unsuccessfull")
}
// Exclusive consumer
messages, err := ch.Consume(queue, "", false, true, false, false, nil)
msg := <-messages
fmt.Println(string(msg.Body))

How to know when the consumer stop consuming

I wanted to create a client that pings to notify the server that it is still online, and when the client stops pinging, the server notifies that there is a problem with the client by sending a message to the terminal.
code for illustrate :
Server :
import (
"log"
"github.com/streadway/amqp"
)
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
}
}
func main() {
conn, err := amqp.Dial("amqp://guest:guest#localhost:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when unused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"logs", // exchange
false,
nil,
)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
forever := make(chan bool)
go func() {
for d := range msgs {
log.Printf(" [x] %s", d.Body)
}
}()
log.Printf(" [*] Waiting for logs. To exit press CTRL+C")
<-forever
}
Client :
package main
import (
"github.com/streadway/amqp"
"log"
"time"
)
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
}
}
func main() {
conn, err := amqp.Dial("amqp://guest:guest#localhost:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
body := "ping"
for i := 0; i < 4; i++ {
err = ch.Publish(
"logs", // exchange
"", // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "text/plain",
Body: []byte(body),
})
time.Sleep(3 * time.Second)
failOnError(err, "Failed to publish a message")
log.Printf(" [x] %s sent to the server", body)
}
}
I was wondering, how can we know when the consumer stop consuming. Maybe with this information i will be able to know when the client is dead ?

Limiting concurrency when processing messages from RabbitMQ

I'm attempting to read URLs from a queue (RabbitMQ) and make a limited number of concurrent HTTP requests i.e. have a pool of 10 workers making concurrent requests to URLs received from the queue (forever).
So far I've implemented a consumer as per the RabbitMQ tutorials:
https://www.rabbitmq.com/tutorials/tutorial-one-go.html
And have tried a number of methods from examples discovered on the web, ending at the example here:
http://jmoiron.net/blog/limiting-concurrency-in-go/
Unfortunately, my current code runs for approximately one minute and then freezes indefinitely. I've tried adding/moving go routines around but I can't seem to get it to work as intended (I'm very new to Go).
Current code:
package main
import (
"fmt"
"log"
"net/http"
"time"
"github.com/Xide/bloom"
"github.com/streadway/amqp"
)
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
var netClient = &http.Client{
Timeout: time.Second * 10,
}
func getRequest(url string) {
//resp, err := http.Get(string(url))
resp, err := netClient.Get(string(url))
if err != nil {
log.Printf("HTTP request error: %s", err)
return
}
fmt.Println("StatusCode:", resp.StatusCode)
fmt.Println(resp.Request.URL)
}
func main() {
bf := bloom.NewDefaultScalable(0.1)
conn, err := amqp.Dial("amqp://127.0.0.1:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
q, err := ch.QueueDeclare(
"urls", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.Qos(
1, // prefetch count
0, // prefetch size
false, //global
)
failOnError(err, "Failed to set Qos")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
forever := make(chan bool)
concurrency := 10
sem := make(chan bool, concurrency)
go func() {
for d := range msgs {
sem <- true
url := string(d.Body)
if bf.Match(url) == false {
bf.Feed(url)
log.Printf("Not seen: %s", d.Body)
go func(url string) {
defer func() { <-sem }()
getRequest(url)
}(url)
} else {
log.Printf("Already seen: %s", d.Body)
}
d.Ack(false)
}
for i := 0; i < cap(sem); i++ {
sem <- true
}
}()
log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
<-forever
}
You're not properly handling your HTTP responses, leading to a growing set of open connections. Try this:
func getRequest(url string) {
resp, err := netClient.Get(string(url))
if err != nil {
log.Printf("HTTP request error: %s", err)
return
}
// Add this bit:
defer func() {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}()
fmt.Println("StatusCode:", resp.StatusCode)
fmt.Println(resp.Request.URL)
}
This, after you finish reading messages from the channel, seems unnecessary and potentially problematic:
for i := 0; i < cap(sem); i++ {
sem <- true
}
Why fill the sem channel after you've read all the messages from the queue? You've added exactly as many messages to the channel as you expect to read from it, so this is pointless at best, and could cause problems if you make the wrong change to the rest of the code.
Unrelated to your issue, but this is redundant:
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
Per the documentation, Fatalf already exits, so the panic will never be called. If you want to log and panic, try log.Panicf, which is designed for that purpose.
You are adding to sem when you get a message, but only removing from sem when you haven't seen a url.
so, once you've "already seen" 10 urls, your app will lock up.
So you need to add <-sem to your else statement that logs "Already seen".
Either way, that's a fairly odd way to do this kind of concurrency.
Here's a version that does this in a more idiomatic way on Play.
Note, in this version, we just spawn 10 goroutines that listen to the rabbit channel.
package main
import (
"fmt"
"log"
"net/http"
"time"
"github.com/Xide/bloom"
"github.com/streadway/amqp"
)
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
}
}
var netClient = &http.Client{
Timeout: time.Second * 10,
}
func getRequest(url string) {
//resp, err := http.Get(string(url))
resp, err := netClient.Get(string(url))
if err != nil {
log.Printf("HTTP request error: %s", err)
return
}
resp.Body.Close()
fmt.Println("StatusCode:", resp.StatusCode)
fmt.Println(resp.Request.URL)
}
func main() {
bf := bloom.NewDefaultScalable(0.1)
conn, err := amqp.Dial("amqp://127.0.0.1:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
q, err := ch.QueueDeclare(
"urls", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.Qos(
1, // prefetch count
0, // prefetch size
false, //global
)
failOnError(err, "Failed to set Qos")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
concurrency := 10
var wg sync.Waitgroup // used to coordinate when they are done, ie: if rabbit conn was closed
for x := 0; x < concurrency; x++ { // spawn 10 goroutines, all reading from the rabbit channel
wg.Add(1)
go func() {
defer wg.Done() // signal that this goroutine is done
for d := range msgs {
url := string(d.Body)
if bf.Match(url) == false {
bf.Feed(url)
log.Printf("Not seen: %s", d.Body)
getRequest(url)
} else {
log.Printf("Already seen: %s", d.Body)
}
d.Ack(false)
}
log.Println("msgs channel closed")
}()
}
log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
wg.Wait() // when all goroutine's exit, the app exits
}

golang websocket how to be notified when browser closed unexpectly

I am writing a websocket service in golang.
The program use gollira websocket to accept ws request, and in each request handler, it listen to rabbitmq queue for messages.
The problem is, when i close browser window, the handler thread is still running, i guess there is an mechanism to be notified when connection disconnected.
I try to listen to channel request.Context().Done(), when i doesn't work.
package main
import (
"fmt"
"log"
"net/http"
"github.com/gorilla/websocket"
"github.com/streadway/amqp"
)
var (
addr = "localhost:9999"
upgrader = websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool { return true },
}
)
var conn *amqp.Connection
func watch(w http.ResponseWriter, r *http.Request) {
ns := r.URL.Query().Get("ns")
if ns == "" {
return
}
c, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Print("upgrade:", err)
return
}
defer c.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"notify", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
ns, // routing key
"dsm_tasks_notify", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
for {
select {
case d := <-msgs:
err = c.WriteMessage(websocket.TextMessage, d.Body)
if err != nil {
log.Println("write:", err)
break
}
case <-r.Context().Done():
log.Println("Disconnect")
return
}
}
}
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
func main() {
var err error
conn, err = amqp.Dial("amqp://guest:guest#localhost:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
http.HandleFunc("/watch", watch)
log.Fatal(http.ListenAndServe(addr, nil))
}
If the browser cleanly closes the connection, then read on the webssocket connection returns an error. Cleanup the websocket connection as you would on any read error.
The application must PING the connection and expect the corresponding PONGs to detect other situations. The chat example shows how to send PINGs and receive PONGs.

Could one connection support multiple channels in go api for rabbitmq?

package main
import (
"fmt"
"github.com/streadway/amqp"
"time"
)
// Every connection should declare the topology they expect
func setup(url, queue string) (*amqp.Connection, *amqp.Channel, error) {
//setup connection
conn, err := amqp.Dial(url)
if err != nil {
return nil, nil, err
}
//build channel in the connection
ch, err := conn.Channel()
if err != nil {
return nil, nil, err
}
//queue declare
if _, err := ch.QueueDeclare(queue, false, true, false, false, nil); err != nil {
return nil, nil, err
}
return conn, ch, nil
}
func main() {
//amqp url
url := "amqp://guest:guest#127.0.0.1:5672";
for i := 1; i <= 2; i++ {
fmt.Println("connect ", i)
//two goroutine
go func() {
//queue name
queue := fmt.Sprintf("example.reconnect.%d", i)
//setup channel in the tcp connection
_, pub, err := setup(url, queue)
if err != nil {
fmt.Println("err publisher setup:", err)
return
}
// Purge the queue from the publisher side to establish initial state
if _, err := pub.QueuePurge(queue, false); err != nil {
fmt.Println("err purge:", err)
return
}
//publish msg
if err := pub.Publish("", queue, false, false, amqp.Publishing{
Body: []byte(fmt.Sprintf("%d", i)),
}); err != nil {
fmt.Println("err publish:", err)
return
}
//keep running
for{
time.Sleep(time.Second * 20)
}
}()
}
//keep running
for {
time.Sleep(time.Second * 20)
}
}
I thought there is only one connection between the program and mq-server,
but there are two connection,one connection can only support one channel,why?
can't the two goroutine share the same tcp connection?
Socket descriptor can share in all threads of a process in the theory.
Why the two goroutine don't share one socket but have their own channel?
The model by hand:
The real model in rabbitmq:
Looking at the source for the library it appears as though you can call conn.Channel() as many times as you like and it creates a new stream of communication over the same connection.
Ok, I tried it, here's a working example... One goroutine, one connection, two channels
I setup the receiver, then send a message, then read from the receiver channel
if you wanted multiple queue's bound in one goroutine, you would call rec.Consume twice and then select across the queues.
package main
import (
"fmt"
"github.com/streadway/amqp"
"os"
)
func main() {
conn, err := amqp.Dial("amqp://localhost")
e(err)
defer conn.Close()
fmt.Println("Connected")
rec, err := conn.Channel()
e(err)
fmt.Println("Setup receiver")
rq, err := rec.QueueDeclare("go-test", false, false, false, false, nil)
e(err)
msgs, err := rec.Consume(rq.Name, "", true, false, false, false, nil)
e(err)
fmt.Println("Setup sender")
send, err := conn.Channel()
e(err)
sq, err := send.QueueDeclare("go-test", false, false, false, false, nil)
e(err)
fmt.Println("Send message")
err = send.Publish("", sq.Name, false, false, amqp.Publishing{
ContentType: "text/plain",
Body: []byte("This is a test"),
})
e(err)
msg := <-msgs
fmt.Println("Received from:", rq, "msg:", string(msg.Body))
}
func e(err error) {
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
Output on my box:
$ go run rmq.go
Connected
Setup receiver
Setup sender
Send message
Received from: {go-test 0 0} msg: This is a test

Resources