Long Spring Framework JMS Client Timeout On Broker Restart - spring

Question

I have a test JMS client application that uses JmsTemplate to send, and DefaultMessageListenerContainer to receive.
The application sends a single message to a queue, and then reads that message from the queue. It does this once a second. Think of it as a "periodic ping" to the broker. My goal is to use this to detect and report the ability to "ping the broker". In this context when the broker is down, I want the ping to fail within a configurable time period (e.g. 2 seconds).
When the system is healthy the typical "ping time" takes about 96 msec.
I then kill and restart the broker and expect the client ping to fail, it does. However the call to convertAndSend takes a very long time to return (15 to 21 seconds). This is much greater than my goal of 2 seconds.
The 15 second timeout can be seen here:
Notice in particular the two exceptions that are caught. The first one (underlined in yellow) is caught by the DefaultMessageListenerContainer. The second one (underlined in red) is one thrown by convertAndSend (caught in pol, see TrackedBroker.poll method in code below).
Once the broker is back (up and running) the "pinger" application again periodically pings the broker in the "healthy system"fashion.
In my current implementation the DefaultMessageListenerContainer, and the JmsTemplate each use their own pooling connection factory. As the connectionFactory construction method (seen below) suggests, I have attempted to configure the factory to shorten the length of timeouts. It would seem that the DefaultMessageListenerContainer is blocking for 15 seconds.
I am looking for suggestions on how to shorten the ping failure timeout from 15 to 2.
I'd prefer to let spring do as much as possible in terms of connection lifecycle management. I only started to muck with things like timeouts, when I was trying to solve this problem. I've dug through spring JMS documentation on JMSTemplate and on DefaultMessageListenerContainer, however because the management of the connection lifecycle to a JMS broker is not simple, its difficult to understand exactly how to resolve this problem.
The following is the code that I am currently using:
TrackedBroker
import org.apache.activemq.artemis.api.core.client.ActiveMQClient;
import org.apache.activemq.artemis.api.core.client.ClientSessionFactory;
import org.apache.activemq.artemis.api.core.client.ServerLocator;
import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jms.UncategorizedJmsException;
import org.springframework.jms.connection.CachingConnectionFactory;
import org.springframework.jms.connection.SingleConnectionFactory;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.listener.DefaultMessageListenerContainer;
import org.springframework.jms.listener.MessageListenerContainer;
import org.springframework.util.ErrorHandler;
import org.springframework.util.backoff.FixedBackOff;
import javax.jms.*;
import java.util.Date;
public class TrackedBroker implements MessageListener, ErrorHandler {
private int POLL_ATTEMPT_FAIL_THRESHOLD = 3;
private boolean alive = false;
private String name;
private String description;
private int pollAttempt = POLL_ATTEMPT_FAIL_THRESHOLD;
private int sequenceNumber = 0;
JmsTemplate template;
MessageListenerContainer messageListenerContainer;
String url;
String user;
String password;
String brokerEndpoint;
private static final Logger log = LoggerFactory.getLogger(TrackedBroker.class);
public TrackedBroker(AppConfig.Broker b, String brokerEndpoint) {
this.name = b.getName();
this.user = b.getUser();
this.password = b.getPassword();
this.brokerEndpoint = brokerEndpoint;
this.description = b.getDescription();
String url = "tcp://"+b.getHost()+":"+b.getPort();
this.url = url;
}
private MessageListenerContainer listenerContainer(String url, String user, String pw, String brokerEndpoint) throws JMSException {
DefaultMessageListenerContainer container = new DefaultMessageListenerContainer();
container.initialize();
container.setConnectionFactory(connectionFactory(url, user, pw));
container.setDestinationName(brokerEndpoint);
container.setReceiveTimeout(1000);
//container.setConcurrency("4");
//container.setConcurrentConsumers(10);
FixedBackOff backoff = new FixedBackOff(200, FixedBackOff.UNLIMITED_ATTEMPTS);
container.setBackOff(backoff);
container.setRecoveryInterval(100);
container.setMessageListener(this);
container.setErrorHandler(this);
return container;
}
private ConnectionFactory connectionFactory(String url, String user, String pw) throws JMSException{
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory();
connectionFactory.setPreAcknowledge(true);
connectionFactory.setBlockOnAcknowledge(false);
connectionFactory.setBlockOnNonDurableSend(false);
connectionFactory.setCallTimeout(500);
connectionFactory.setCallFailoverTimeout(500);
connectionFactory.setClientFailureCheckPeriod(500);
connectionFactory.setCallFailoverTimeout(400);
//connectionFactory.setConnectionTTL(3000);
connectionFactory.setBrokerURL(url);
connectionFactory.setPassword(pw);
connectionFactory.setUser(user);
CachingConnectionFactory caching_connection_factory = new CachingConnectionFactory(connectionFactory);
return caching_connection_factory;
}
private JmsTemplate jmsTemplate(String url, String user, String pw) throws JMSException{
JmsTemplate template = new JmsTemplate();
template.setConnectionFactory(connectionFactory(url, user, pw));
template.setDeliveryPersistent(false);
//template.setDeliveryMode(DeliveryMode.NON_PERSISTENT);
return template;
}
public void init() throws JMSException {
setTemplate(jmsTemplate(url, user, password));
setMessageListenerContainer(listenerContainer(url, user, password, brokerEndpoint));
getMessageListenerContainer().start();
}
public void sendMessage(JmsTemplate jmsTmpl, String message) {
String label1 = "Broker Convert & Send" + getName();
Date date1 = DeltaLogger.logStart(label1);
jmsTmpl.convertAndSend(brokerEndpoint, message);
DeltaLogger.logEnd(label1, date1);
}
public void poll() {
incrementPollAttempt();
incrementSequenceNumber();
String probeMsg = "Probe Broker " + getName() + " Yo " + getSequenceNumber();
try {
String label1 = "Broker Send" + getName();
Date date1 = DeltaLogger.logStart(label1);
sendMessage(getTemplate(), probeMsg);
DeltaLogger.logEnd(label1, date1);
} catch (UncategorizedJmsException e) {
log.error("JMS Exception " + e.getLocalizedMessage());
}
log.info("My current poll attempts " + getName() + ": " + getPollAttempt());
if(getPollAttempt() > POLL_ATTEMPT_FAIL_THRESHOLD) {
setAlive(false);
} else {
setAlive(true);
}
}
public boolean isAlive() {
return alive;
}
private void setAlive(boolean alive) {
this.alive = alive;
}
public String getName() {
return name;
}
private void setName(String name) {
this.name = name;
}
public int getPollAttempt() {
return pollAttempt;
}
private void setPollAttempt(int pollAttempt) {
this.pollAttempt = pollAttempt;
}
private void incrementPollAttempt() {
this.pollAttempt++;
}
private void clearPollAttempt() {
this.pollAttempt = 0;
}
public JmsTemplate getTemplate() {
return template;
}
private void setTemplate(JmsTemplate template) {
this.template = template;
}
private void setSequenceNumber(int sequenceNumber) {
this.sequenceNumber = sequenceNumber;
}
public int getSequenceNumber() {
return sequenceNumber;
}
private void incrementSequenceNumber() {
this.sequenceNumber++;
}
public MessageListenerContainer getMessageListenerContainer() {
return messageListenerContainer;
}
private void setMessageListenerContainer(MessageListenerContainer messageListenerContainer) {
this.messageListenerContainer = messageListenerContainer;
}
public String getDescription() {
return description;
}
private void setDescription(String description) {
this.description = description;
}
#Override
public void onMessage(Message message) {
if (message instanceof TextMessage) {
try {
String text = ((TextMessage) message).getText();
log.info("Received " + text);
clearPollAttempt();
} catch (JMSException e) {
e.printStackTrace();
}
}
}
#Override
public void handleError(Throwable throwable) {
log.error("Error" + throwable.getLocalizedMessage());
}
}
I currently load the following configuration (configprops.json):
{
"brokers": [
{"name": "artemis0", "description": "tonga", "host": "10.93.97.202", "port": 32192, "user": "artemis", "password": "aaa"},
{"name": "artemis1", "description": "spain", "host": "10.93.103.236", "port": 32192, "user": "artemis", "password": "aaa"}
]
}
JsonPropertySourceFactory
import com.fasterxml.jackson.databind.ObjectMapper;
import com.saab.ims.smart_proxy.controller.ScheduledTasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.MapPropertySource;
import org.springframework.core.env.PropertySource;
import org.springframework.core.io.support.EncodedResource;
import org.springframework.core.io.support.PropertySourceFactory;
import java.io.IOException;
import java.util.Map;
public class JsonPropertySourceFactory
implements PropertySourceFactory {
private static final Logger log = LoggerFactory.getLogger(JsonPropertySourceFactory.class);
#Override
public PropertySource<?> createPropertySource(
String name, EncodedResource resource)
throws IOException {
Map readValue = new ObjectMapper()
.readValue(resource.getInputStream(), Map.class);
return new MapPropertySource("json-property", readValue);
}
}
AppConfig
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
#Component
#PropertySource(
value = "classpath:configprops.json",
factory = JsonPropertySourceFactory.class)
#ConfigurationProperties
public class AppConfig {
private static final Logger log = LoggerFactory.getLogger(AppConfig.class);
private int port;
private boolean resend;
private String host;
ObjectMapper mapper = new ObjectMapper();
public int getPort() {
return port;
}
public void setResend(boolean resend) {
this.resend = resend;
}
public boolean isResend() {
return resend;
}
public void setPort(int port) {
this.port = port;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
private List<String> topics;
public List<String> getTopics() {
return topics;
}
public void setTopics(List<String> topics) {
this.topics = topics;
}
private List<Broker> brokers ;
public List<Broker> getBrokers() {
return mapper.convertValue(brokers,
new TypeReference<List<Broker>>(){});
}
public void setBrokers(List<Broker> brokers) {
log.info("Broker is " + brokers);
this.brokers = brokers;
}
public static class Broker {
private String name;
private String description;
private String host;
private int port;
private String user;
private String password;
public String getName() {
return name;
}
public String getDescription() {
return description;
}
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}
}
ScheduledTasks
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.listener.MessageListenerContainer;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
#Component
public class ScheduledTasks {
private static final Logger log = LoggerFactory.getLogger(ScheduledTasks.class);
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
private long probeBrokerCnt = 0;
private long probeProxyCnt = 0;
private ArrayList<TrackedBroker> trackedBrokerList = new ArrayList<TrackedBroker>();
#Autowired
DispatcherService dispatcherService;
#Autowired
HaProxyService haproxyService;
#Autowired
private AppConfig appConfig;
#PostConstruct
public void init() {
List<AppConfig.Broker> brokerList =appConfig.getBrokers();
for(int i = 0; i < brokerList.size(); i++) {
AppConfig.Broker b = brokerList.get(i);
log.info("My cool broker " + b.getName() +
" Host " + b.getHost() + " Port " + b.getPort() +
" User " + b.getUser() + " PW " + b.getPassword());
TrackedBroker tb = new TrackedBroker(b, "Q.Proxy.Probe");
try {
tb.init();
} catch (JMSException e) {
log.error("Failed to create JMS Template " + e);
}
trackedBrokerList.add(tb);
}
}
#Scheduled(fixedRate = 5000)
public void reportCurrentTime() {
log.info("The time is now {}", dateFormat.format(new Date()));
}
#Scheduled(fixedRate = 1000)
public void probeBroker() {
Date date1 = new Date();
log.info("Starting Probe ");
for(TrackedBroker tb : trackedBrokerList) {
tb.poll();
log.info("Broker " + tb.getName() + " Desc " + tb.getDescription() + " Alive " + Boolean.toString(tb.isAlive()));
}
Date date2 = new Date();
long difference = date2.getTime() - date1.getTime();
log.info("Ending Probe- Duration(seconds): " + Double.toString(difference/1000.0));
//dispatcherService.sendMessage(probeMsg);
}
}