Reply messages timing out - Spring AMQP - spring

Question

In our application around 1% of the transactions times out daily.
We clearly can see in the log that messages is being processed and sent back using reply-to mechanism but client not receiving the message and wait till the timeout then throws an error.
I'm unable to identify the issue that why these messages getting lost occasionally. Any help much appreciated.
Clint -> Rabbit -> Server -> Rabbit -> Client (timeout - 1% of transactions)
log lines:
client:
2021-11-28 23:03:43.317 WARN <sending msg to vhost <IP> with correlationId: 5cc5c40b-7193-1cf9-c66f-7c5d72901bdf , message properties correlationId: 10976>
Server:
2021-11-28 23:03:43.318 WARN <Received msg with correlationId: 5cc5c40b-7193-1cf9-c66f-7c5d72901bdf , message properties correlationId: 10976>
Server:
2021-11-28 23:03:57.830 WARN <sending msg to vhost <IP> with correlationId: 112f5771-93b9-6f8f-a353-b9a79bdd1438 , message properties correlationId: 10976>
client:
2021-11-28 23:04:43.317 ERROR - <An exception has occured: No reply received from 'assign' with arguments '[]' - perhaps a timeout in the template? org.springframework.remoting.RemoteProxyFailureException: No reply received from 'assign' with arguments '[]' - perhaps a timeout in the template?>
Client and Listener configuration:
<!-- client config -->
<rabbit:queue id="application.queue" name="API.request.queue"
declared-by="rabbitAdminConnectionFactory1"/>
<rabbit:direct-exchange name="#{ API +'.RequestDirectExchange'}" id="requestDirectExchange"
declared-by="rabbitAdminConnectionFactory1">
<rabbit:bindings>
<rabbit:binding queue="application.queue" key="API"/>
</rabbit:bindings>
</rabbit:direct-exchange>
<bean id="amqpTemplate" class="com.api.APIRabbitTemplate">
<property name="connectionFactory" ref="rabbitConnectionFactory"/>
<property name="routingKey" value="API"/>
<property name="exchange" value="#{ API +'.RequestDirectExchange'}"/>
<property name="queue" value="API.request.queue"/>
<property name="messageConverter" ref="rmessageConverter"/>
<property name="replyTimeout" value="60000"/>
<property name="receiveTimeout" value="60000"/>
<property name="retryTemplate" ref="retryTemplate"/>
</bean>
<!-- server listener config -->
<bean id="remotingAmqpTemplate1" class="com.API.APIRabbitTemplate">
<property name="connectionFactory" ref="rabbitConnectionFactory"/>
<property name="messageConverter" ref="rmessageConverter"/>
<property name="queue" value="$app{appName}.request.queue"/>
<property name="retryTemplate" ref="retryTemplate1"/>
</bean>
<bean id="amqpRemotingListener1" class="com.API.remoting.amqpRemotingListener">
<property name="amqpTemplate" ref="remotingAmqpTemplate1"/>
</bean>
<rabbit:listener-container connection-factory="rabbitConnectionFactory" auto-startup="false"
requeue-rejected="false" task-executor="taskExecutor"
concurrency="6" max-concurrency="6"
acknowledge="auto" prefetch="1"
transaction-size="1" channel-transacted="true">
<rabbit:listener ref="amqpRemotingListener1" queue-names="API.request.queue"
admin="rabbitAdminConnectionFactory1"/>
</rabbit:listener-container>
<rabbit:connection-factory id="rabbitConnectionFactory"
thread-factory="tfCommon1"
connection-factory="clientConnectionFactory"
addresses="$amqp{connectionURL1}"
username="$amqp{username}"
password="$amqp{password}"
channel-cache-size="$amqp{listeners.session.cache.size}"/>
<util:properties id="spring.amqp.global.properties">
<prop key="smlc.missing.queues.fatal">false</prop>
</util:properties>
EDIT:
public class ARabbitTemplate extends RabbitTemplate {
protected static Logger log = Logger.getLogger( AARabbitTemplate.class );
#Override
public <T> T execute(ChannelCallback<T> action) {
try {
return super.execute(action);
}catch (AmqpException ex){
//log
throw ex;
}
}
#Override
public void doSend(Channel channel, String exchange, String routingKey, Message message,
boolean mandatory, CorrelationData correlationData) throws Exception {
try {
super.doSend(channel, exchange, routingKey, message, mandatory, correlationData);
if(channel != null && channel.getConnection() != null && channel.getConnection().getAddress() != null) {
String IP = channel.getConnection().getAddress().getHostAddress();
String correlationId = message.getMessageProperties().getHeaders().get(MessagingConsts.CORRELATION_ID);
String message_CorrelationId = message.getMessageProperties().getCorrelationId();
log("sending msg to vhost " + IP + " with correlationId: " + correlationId + " , message properties correlationId: " + message_CorrelationId);
}
} catch (Exception ex) {
log.error("got exception while sending msg to ampq "+ ex.getLocalizedMessage());
throw ex;
}
}
#Override
public <R, S> boolean receiveAndReply(String queueName, ReceiveAndReplyCallback<R, S> callback,
ReplyToAddressCallback<S> replyToAddressCallback) throws AmqpException {
try {
return super.receiveAndReply(queueName, callback, replyToAddressCallback);
} catch (AmqpException amqpEx) {
throw amqpEx;
}
}
#Override
public Message receive() throws AmqpException {
try {
return super.receive();
} catch (AmqpException amqpEx) {
throw amqpEx;
}
}
#Override
public void onMessage(Message message) {
super.onMessage(message);
}
}
public class AAmqpRemotingListener implements MessageListener {
private AmqpTemplate amqpTemplate;
#Autowired
private MessageConverter messageConverter;
#Override
public void onMessage(Message message) {
String correlationId = message.getMessageProperties().getHeaders().get(MessagingConsts.CORRELATION_ID);
String message_CorrelationId = message.getMessageProperties().getCorrelationId();
log("Received msg with correlationId: " + correlationId + " , message properties correlationId: " + message_CorrelationId);
//rest of the code
//exporter.onMessage(message);
}
}
//Message Converter
public class SpringAmqpMessageConverter implements MessageConverter {
#Autowired
protected ObjectSerializer serializer;
#Override
public Message toMessage(Object object, MessageProperties messageProperties) throws MessageConversionException {
byte[] data = serializer.writeObject(object);
// build message
MessageBuilder messageBuilder = MessageBuilder.withBody(data);
String correlationId = context.getCorrelationId();
if (StringUtils.isEmpty(correlationId)) {
correlationId = (new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong())).toString();
}
messageBuilder.setHeader(MessagingConsts.CORRELATION_ID, correlationId);
messageBuilder.copyHeaders(messageProperties.getHeaders());
messageBuilder.setCorrelationIdIfAbsent(correlationId);
messageBuilder.setDeliveryModeIfAbsentOrDefault(MessageDeliveryMode.NON_PERSISTENT);
Message message = messageBuilder.build();
return message;
}
#Override
public Object fromMessage(Message message) throws MessageConversionException {
//
}
}