RabbitMQ: Channel shutdown: connection error - spring

Question

I've setup rabbitmq with spring boot. Everything seems to be working fine in my local environment. But when I deploy it to production I'm getting this error:
31-05-2019 08:26:56.430 [ajp-bio-8009-exec-2] INFO org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createBareConnection - Attempting to connect to: localhost:5672
31-05-2019 08:26:56.433 [AMQP Connection 127.0.0.1:5672] ERROR org.springframework.amqp.rabbit.connection.CachingConnectionFactory.log - Channel shutdown: connection error; protocol method: #method<connection.close>(reply-code=541, reply-text=INTERNAL_ERROR, class-id=0, method-id=0)
31-05-2019 08:26:56.434 [AMQP Connection 127.0.0.1:5672] WARN com.rabbitmq.client.impl.ForgivingExceptionHandler.log - An unexpected connection driver error occured (Exception message: Connection reset)
31-05-2019 08:26:56.452 [ajp-bio-8009-exec-2] INFO org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createBareConnection - Created new connection: connectionFactory#8139c95:2/SimpleConnection#4f0210ae [delegate=amqp://admin#127.0.0.1:5672/, localPort= 47206]
31-05-2019 08:26:56.457 [ajp-bio-8009-exec-2] ERROR com.cinch.gch.core.config.GchExceptionHandler.handleAllOtherException - ExceptionHandler
org.springframework.amqp.AmqpIOException: java.io.IOException
at org.springframework.amqp.rabbit.support.RabbitExceptionTranslator.convertRabbitAccessException(RabbitExceptionTranslator.java:71)
at org.springframework.amqp.rabbit.connection.RabbitAccessor.convertRabbitAccessException(RabbitAccessor.java:113)
at org.springframework.amqp.rabbit.core.RabbitTemplate.doExecute(RabbitTemplate.java:1473)
at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:1423)
at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:1399)
at org.springframework.amqp.rabbit.core.RabbitAdmin.declareExchange(RabbitAdmin.java:180)
at com.cinch.gch.mailq.queue.ProducerConfiguration.<init>(ProducerConfiguration.java:41)
at com.cinch.gch.mailq.queue.MQQueue.init(MQQueue.java:39)
at com.cinch.gch.mailq.queue.MQQueue.<init>(MQQueue.java:25)
at com.cinch.gch.mailq.service.MailService.send(MailService.java:42)
at com.cinch.gch.mailq.controller.MailController.sendMails(MailController.java:32)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:854)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:765)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.springframework.boot.web.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:130)
at org.springframework.boot.web.support.ErrorPageFilter.access$000(ErrorPageFilter.java:66)
at org.springframework.boot.web.support.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.boot.web.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408)
at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:193)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:607)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:313)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: null
at com.rabbitmq.client.impl.AMQChannel.wrap(AMQChannel.java:124)
at com.rabbitmq.client.impl.AMQChannel.wrap(AMQChannel.java:120)
at com.rabbitmq.client.impl.AMQChannel.exnWrappingRpc(AMQChannel.java:142)
at com.rabbitmq.client.impl.ChannelN.exchangeDeclare(ChannelN.java:773)
at com.rabbitmq.client.impl.ChannelN.exchangeDeclare(ChannelN.java:53)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.amqp.rabbit.connection.CachingConnectionFactory$CachedChannelInvocationHandler.invoke(CachingConnectionFactory.java:1029)
at com.sun.proxy.$Proxy564.exchangeDeclare(Unknown Source)
at org.springframework.amqp.rabbit.core.RabbitAdmin.declareExchanges(RabbitAdmin.java:630)
at org.springframework.amqp.rabbit.core.RabbitAdmin.access$000(RabbitAdmin.java:72)
at org.springframework.amqp.rabbit.core.RabbitAdmin$1.doInRabbit(RabbitAdmin.java:184)
at org.springframework.amqp.rabbit.core.RabbitTemplate.doExecute(RabbitTemplate.java:1467)
... 52 common frames omitted
Caused by: com.rabbitmq.client.ShutdownSignalException: connection error; protocol method: #method<connection.close>(reply-code=541, reply-text=INTERNAL_ERROR, class-id=0, method-id=0)
at com.rabbitmq.utility.ValueOrException.getValue(ValueOrException.java:66)
at com.rabbitmq.utility.BlockingValueOrException.uninterruptibleGetValue(BlockingValueOrException.java:36)
at com.rabbitmq.client.impl.AMQChannel$BlockingRpcContinuation.getReply(AMQChannel.java:443)
at com.rabbitmq.client.impl.AMQChannel.privateRpc(AMQChannel.java:263)
at com.rabbitmq.client.impl.AMQChannel.exnWrappingRpc(AMQChannel.java:136)
... 64 common frames omitted
31-05-2019 08:26:57.434 [AMQP Connection 127.0.0.1:5672] ERROR org.springframework.amqp.rabbit.connection.CachingConnectionFactory.log - Channel shutdown: connection error; protocol method: #method<connection.close>(reply-code=320, reply-text=CONNECTION_FORCED - broker forced connection closure with reason 'shutdown', class-id=0, method-id=0)
31-05-2019 08:26:57.444 [AMQP Connection 127.0.0.1:5672] WARN com.rabbitmq.client.impl.ForgivingExceptionHandler.log - An unexpected connection driver error occured (Exception message: Connection reset)
Anyone have any idea what is about? Not posting code since it's working in local environment, and I believe it's because I'm missing something in the server side. If you think otherwise, let me know.
Edit
Error occurs when I'm trying to create queues dynamically and add consumers. It seems the queues are created in the management console. but the listeners are not added.
Edit
Rabbitmq logs
=ERROR REPORT==== 31-May-2019::09:03:17 ===
** Generic server <0.627.0> terminating
** Last message in was {'$gen_cast',
{method,
{'exchange.declare',0,<<"mailq-exchange">>,
<<"x-delayed-message">>,false,true,false,
false,false,
[{<<"x-delayed-type">>,longstr,
<<"topic">>}]},
none,noflow}}
** When Server state == {ch,running,rabbit_framing_amqp_0_9_1,1,<0.618.0>,
<0.625.0>,<0.618.0>,
<<"127.0.0.1:47239 -> 127.0.0.1:5672">>,
{lstate,<0.626.0>,false},
none,1,
{[],[]},
{user,<<"admin">>,
[administrator],
[{rabbit_auth_backend_internal,none}]},
<<"/">>,<<"q:17252337684797619970874926144232">>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
erlang},
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{set,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
<0.620.0>,
{state,fine,5000,#Ref<0.0.0.30310>},
false,1,
{{0,nil},{0,nil}},
[],
{{0,nil},{0,nil}},
[{<<"exchange_exchange_bindings">>,bool,true},
{<<"connection.blocked">>,bool,true},
{<<"authentication_failure_close">>,bool,true},
{<<"basic.nack">>,bool,true},
{<<"publisher_confirms">>,bool,true},
{<<"consumer_cancel_notify">>,bool,true}],
none,0,none,flow,[]}
** Reason for termination ==
** {function_clause,
[{rabbit_exchange_type_delayed_message,validate,
[{exchange,
{resource,<<"/">>,exchange,<<"mailq-exchange">>},
'x-delayed-message',true,false,false,
[{<<"x-delayed-type">>,longstr,<<"topic">>}],
undefined,undefined,
{[],[]}}],
[{file,"src/rabbit_exchange_type_delayed_message.erl"},{line,61}]},
{rabbit_exchange,declare,6,
[{file,"src/rabbit_exchange.erl"},{line,165}]},
{rabbit_channel,handle_method,3,
[{file,"src/rabbit_channel.erl"},{line,1234}]},
{rabbit_channel,handle_cast,2,
[{file,"src/rabbit_channel.erl"},{line,470}]},
{gen_server2,handle_msg,2,[{file,"src/gen_server2.erl"},{line,1047}]},
{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}
=ERROR REPORT==== 31-May-2019::09:03:17 ===
Error on AMQP connection <0.618.0> (127.0.0.1:47239 -> 127.0.0.1:5672 - connectionFactory#8139c95:4, vhost: '/', user: 'admin', state: running), channel 1:
{function_clause,
[{rabbit_exchange_type_delayed_message,validate,
[{exchange,
{resource,<<"/">>,exchange,<<"mailq-exchange">>},
'x-delayed-message',true,false,false,
[{<<"x-delayed-type">>,longstr,<<"topic">>}],
undefined,undefined,
{[],[]}}],
[{file,"src/rabbit_exchange_type_delayed_message.erl"},{line,61}]},
{rabbit_exchange,declare,6,[{file,"src/rabbit_exchange.erl"},{line,165}]},
{rabbit_channel,handle_method,3,
[{file,"src/rabbit_channel.erl"},{line,1234}]},
{rabbit_channel,handle_cast,2,
[{file,"src/rabbit_channel.erl"},{line,470}]},
{gen_server2,handle_msg,2,[{file,"src/gen_server2.erl"},{line,1047}]},
{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}
=WARNING REPORT==== 31-May-2019::09:03:17 ===
Non-AMQP exit reason '{function_clause,
[{rabbit_exchange_type_delayed_message,validate,
[{exchange,
{resource,<<"/">>,exchange,<<"mailq-exchange">>},
'x-delayed-message',true,false,false,
[{<<"x-delayed-type">>,longstr,<<"topic">>}],
undefined,undefined,
{[],[]}}],
[{file,
"src/rabbit_exchange_type_delayed_message.erl"},
{line,61}]},
{rabbit_exchange,declare,6,
[{file,"src/rabbit_exchange.erl"},{line,165}]},
{rabbit_channel,handle_method,3,
[{file,"src/rabbit_channel.erl"},{line,1234}]},
{rabbit_channel,handle_cast,2,
[{file,"src/rabbit_channel.erl"},{line,470}]},
{gen_server2,handle_msg,2,
[{file,"src/gen_server2.erl"},{line,1047}]},
{proc_lib,wake_up,3,
[{file,"proc_lib.erl"},{line,249}]}]}'
=INFO REPORT==== 31-May-2019::09:03:17 ===
accepting AMQP connection <0.1308.0> (127.0.0.1:47419 -> 127.0.0.1:5672)
=INFO REPORT==== 31-May-2019::09:03:17 ===
closing AMQP connection <0.618.0> (127.0.0.1:47239 -> 127.0.0.1:5672 - connectionFactory#8139c95:4, vhost: '/', user: 'admin')
=INFO REPORT==== 31-May-2019::09:03:17 ===
Connection <0.1308.0> (127.0.0.1:47419 -> 127.0.0.1:5672) has a client-provided name: connectionFactory#8139c95:5
=INFO REPORT==== 31-May-2019::09:03:17 ===
connection <0.1308.0> (127.0.0.1:47419 -> 127.0.0.1:5672 - connectionFactory#8139c95:5): user 'admin' authenticated and granted access to vhost '/'
=ERROR REPORT==== 31-May-2019::09:03:28 ===
** Generic server <0.1316.0> terminating
** Last message in was {'$gen_cast',
{method,
{'exchange.declare',0,<<"mailq-exchange">>,
<<"x-delayed-message">>,false,true,false,
false,false,
[{<<"x-delayed-type">>,longstr,
<<"topic">>}]},
none,noflow}}
** When Server state == {ch,running,rabbit_framing_amqp_0_9_1,1,<0.1308.0>,
<0.1314.0>,<0.1308.0>,
<<"127.0.0.1:47419 -> 127.0.0.1:5672">>,
{lstate,<0.1315.0>,false},
none,1,
{[],[]},
{user,<<"admin">>,
[administrator],
[{rabbit_auth_backend_internal,none}]},
<<"/">>,<<"q:20450439579894538978049880093722">>,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{state,
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[],[]}}},
erlang},
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{dict,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
{set,0,16,16,8,80,48,
{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[]},
{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],
[],[]}}},
<0.1309.0>,
{state,fine,5000,#Ref<0.0.0.30583>},
false,1,
{{0,nil},{0,nil}},
[],
{{0,nil},{0,nil}},
[{<<"exchange_exchange_bindings">>,bool,true},
{<<"connection.blocked">>,bool,true},
{<<"authentication_failure_close">>,bool,true},
{<<"basic.nack">>,bool,true},
{<<"publisher_confirms">>,bool,true},
{<<"consumer_cancel_notify">>,bool,true}],
none,0,none,flow,[]}
** Reason for termination ==
** {function_clause,
[{rabbit_exchange_type_delayed_message,validate,
[{exchange,
{resource,<<"/">>,exchange,<<"mailq-exchange">>},
'x-delayed-message',true,false,false,
[{<<"x-delayed-type">>,longstr,<<"topic">>}],
undefined,undefined,
{[],[]}}],
[{file,"src/rabbit_exchange_type_delayed_message.erl"},{line,61}]},
{rabbit_exchange,declare,6,
[{file,"src/rabbit_exchange.erl"},{line,165}]},
{rabbit_channel,handle_method,3,
[{file,"src/rabbit_channel.erl"},{line,1234}]},
{rabbit_channel,handle_cast,2,
[{file,"src/rabbit_channel.erl"},{line,470}]},
{gen_server2,handle_msg,2,[{file,"src/gen_server2.erl"},{line,1047}]},
{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}
=ERROR REPORT==== 31-May-2019::09:03:28 ===
Error on AMQP connection <0.1308.0> (127.0.0.1:47419 -> 127.0.0.1:5672 - connectionFactory#8139c95:5, vhost: '/', user: 'admin', state: running), channel 1:
{function_clause,
[{rabbit_exchange_type_delayed_message,validate,
[{exchange,
{resource,<<"/">>,exchange,<<"mailq-exchange">>},
'x-delayed-message',true,false,false,
[{<<"x-delayed-type">>,longstr,<<"topic">>}],
undefined,undefined,
{[],[]}}],
[{file,"src/rabbit_exchange_type_delayed_message.erl"},{line,61}]},
{rabbit_exchange,declare,6,[{file,"src/rabbit_exchange.erl"},{line,165}]},
{rabbit_channel,handle_method,3,
[{file,"src/rabbit_channel.erl"},{line,1234}]},
{rabbit_channel,handle_cast,2,
[{file,"src/rabbit_channel.erl"},{line,470}]},
{gen_server2,handle_msg,2,[{file,"src/gen_server2.erl"},{line,1047}]},
{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,249}]}]}
=WARNING REPORT==== 31-May-2019::09:03:28 ===
Non-AMQP exit reason '{function_clause,
[{rabbit_exchange_type_delayed_message,validate,
[{exchange,
{resource,<<"/">>,exchange,<<"mailq-exchange">>},
'x-delayed-message',true,false,false,
[{<<"x-delayed-type">>,longstr,<<"topic">>}],
undefined,undefined,
{[],[]}}],
[{file,
"src/rabbit_exchange_type_delayed_message.erl"},
{line,61}]},
{rabbit_exchange,declare,6,
[{file,"src/rabbit_exchange.erl"},{line,165}]},
{rabbit_channel,handle_method,3,
[{file,"src/rabbit_channel.erl"},{line,1234}]},
{rabbit_channel,handle_cast,2,
[{file,"src/rabbit_channel.erl"},{line,470}]},
{gen_server2,handle_msg,2,
[{file,"src/gen_server2.erl"},{line,1047}]},
{proc_lib,wake_up,3,
[{file,"proc_lib.erl"},{line,249}]}]}'
=INFO REPORT==== 31-May-2019::09:03:28 ===
closing AMQP connection <0.1308.0> (127.0.0.1:47419 -> 127.0.0.1:5672 - connectionFactory#8139c95:5, vhost: '/', user: 'admin')
=INFO REPORT==== 31-May-2019::09:03:28 ===
accepting AMQP connection <0.1330.0> (127.0.0.1:47421 -> 127.0.0.1:5672)
=INFO REPORT==== 31-May-2019::09:03:29 ===
Connection <0.1330.0> (127.0.0.1:47421 -> 127.0.0.1:5672) has a client-provided name: connectionFactory#8139c95:6
=INFO REPORT==== 31-May-2019::09:03:29 ===
connection <0.1330.0> (127.0.0.1:47421 -> 127.0.0.1:5672 - connectionFactory#8139c95:6): user 'admin' authenticated and granted access to vhost '/'