How to Configure JMS Listener with logging Sleuth Trace ID in SQS Communication - spring-boot

I am using spring JMS to send and recieve message to AWS SQS. Below is my JMS configuration class.
I am using spring sleuth to log trace id.
What i want to achive is the trace id when logged should be same in producer class and in consumer class.
But i am seeing different trace id. How to solve this issue?
import javax.jms.ConnectionFactory;
import javax.jms.Session;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.jms.annotation.EnableJms;
import org.springframework.jms.config.DefaultJmsListenerContainerFactory;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.support.converter.MappingJackson2MessageConverter;
import org.springframework.jms.support.converter.MessageConverter;
import org.springframework.jms.support.converter.MessageType;
import org.springframework.jms.support.destination.DynamicDestinationResolver;
import com.amazon.sqs.javamessaging.SQSConnectionFactory;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
import brave.Tracing;
import brave.jms.JmsTracing;
import brave.messaging.MessagingTracing;
/**
* Description of JMSConfig
*
* #author
* #version Nov 25, 2020
*/
#Configuration
#EnableJms
#Profile({ "Generic" })
public class JmsConfig {
#Autowired
private Tracing tracing;
SQSConnectionFactory connectionFactory = SQSConnectionFactory.builder().withRegion(Region.getRegion(
Regions.US_EAST_1)).withAWSCredentialsProvider(new DefaultAWSCredentialsProviderChain()).build();
#Bean
public DefaultJmsListenerContainerFactory jmsListenerContainerFactory() {
ConnectionFactory tracingConnectionFactory = getConnectionFactoryWrappedWithTracing(connectionFactory);
DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory();
factory.setConnectionFactory(tracingConnectionFactory);
factory.setDestinationResolver(new DynamicDestinationResolver());
factory.setConcurrency("3-10");
factory.setSessionAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
factory.setMessageConverter(messageConverter());
return factory;
}
#Bean
public MessageConverter messageConverter() {
Jackson2ObjectMapperBuilder builder = new Jackson2ObjectMapperBuilder();
builder.serializationInclusion(JsonInclude.Include.NON_EMPTY);
builder.dateFormat(new ISO8601DateFormat());
org.springframework.jms.support.converter.MappingJackson2MessageConverter mappingJackson2MessageConverter = new MappingJackson2MessageConverter();
mappingJackson2MessageConverter.setObjectMapper(builder.build());
mappingJackson2MessageConverter.setTargetType(MessageType.TEXT);
mappingJackson2MessageConverter.setTypeIdPropertyName("documentType");
return mappingJackson2MessageConverter;
}
#Bean
public JmsTemplate defaultJmsTemplate() {
JmsTemplate jmsTemplate = new JmsTemplate(this.connectionFactory);
jmsTemplate.setMessageConverter(messageConverter());
return jmsTemplate;
}
#Bean
public ConnectionFactory getConnectionFactoryWrappedWithTracing(SQSConnectionFactory sqsConnectionFactory) {
MessagingTracing messagingTracing = MessagingTracing.newBuilder(tracing).build();
JmsTracing jmsTracing = JmsTracing.create(messagingTracing);
ConnectionFactory tracingConnectionFactory = jmsTracing.connectionFactory(sqsConnectionFactory);
return tracingConnectionFactory;
}
}
MEssage sender method
public String postMessage(String message) {
log.info("Sending message");
jmsTemplate.convertAndSend(queueName, new User(message));
return message;
}
Message reciever
#JmsListener(destination = "${app.queue.name}", containerFactory = "jmsListenerContainerFactory")
public void receiveMessage(User message) throws JsonProcessingException {
log.info("Received message: " + message);
log.info("Received message: {}", message.getName());
}
When i try to send message i can see different trace id 676d14076ee1cbb7 for producer and 32mc3464298c2503b35 for consumer. Ideally i want trace id to be same.
13:37:12 INFO cloud-messaging-service 676d14076ee1cbb7 GenericMessageProducer: Sending message
13:37:14 INFO cloud-messaging-service 676d14076ee1cbb7 SQSMessageProducer: Message sent to SQS with SQS-assigned messageId: 443f6d62-ac20-406a-9ab4-be4bfb1db8d3
13:37:14 INFO cloud-messaging-service 676d14076ee1cbb7 SQSSession: Shutting down SessionCallBackScheduler executor
13:37:21 INFO cloud-messaging-service c3464298c2503b35 GenericMessageConsumer: Received message: User{name=viyaan}
13:37:21 INFO cloud-messaging-service c3464298c2503b35 GenericMessageConsumer: Received message: viyaan
POM Dependency
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>${aws-java-sdk}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>amazon-sqs-java-messaging-lib</artifactId>
<version>${amazon-sqs-java-messaging-lib}</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jms</artifactId>
</dependency>
<dependency>
<groupId>io.zipkin.aws</groupId>
<artifactId>brave-instrumentation-aws-java-sdk-sqs</artifactId>
<version>${brave-instrumentation-aws-java-sdk-sqs}</version>
</dependency>

I had to tweak JMSTemplate, i have to add tracingConnectionFactory in JMsTemplate to make it work
#Bean
public JmsTemplate defaultJmsTemplate() {
ConnectionFactory tracingConnectionFactory = getConnectionFactoryWrappedWithTracing(sqsConnectionFactory());
JmsTemplate jmsTemplate = new JmsTemplate(tracingConnectionFactory);
jmsTemplate.setMessageConverter(messageConverter());
return jmsTemplate;
}

Related

Message grouping with Artemis and Spring JMS not working when using transacted sessions

Message grouping doesn't appear to be working
My Producer application sends the message to the queue via JMS MessageProducer after setting the string property JMSXGroupID to 'product=paper'
My producer application sends another message the same way also with 'product=paper'.
I can see both messages in the queue when I browse that message's headers in the Artemis UI. _AMQ_GROUP_ID has a value of 'product=paper' in both. JMSXGroupID is absent.
When I debug my listener application which uses Spring JMS with a concurrency of 15-15 (15 min 15 max) I can see both messages come through logged under different listener containers. When I look at the map of headers for each, _AMQ_GROUP_ID is absent and JMSXGroupID has a value of null instead of 'product=paper'.
Why isn't message grouping with group id working? Does it have to do with the fact that Artemis didn't translate _AMQ_GROUP_ID back to JMSXGroupID? Or is Spring JMS not registering its multiple consumer threads as different consumers for the broker to see multiple consumers?
Edit:
I was able to get message grouping to work in my application by commenting out lines having to do with using transacted sessions from my container factory bean method. It seems to have to do with using transacted sessions.
Edit2:
Here's a self contained application running against a local standalone Artemis broker (version 2.10.1) and using Spring Boot 2.2.0:
GroupidApplication (spring boot application and beans):
package com.reproduce.groupid;
import java.util.HashMap;
import java.util.Map;
import javax.jms.ConnectionFactory;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.Session;
import org.apache.activemq.artemis.api.core.TransportConfiguration;
import org.apache.activemq.artemis.api.jms.ActiveMQJMSClient;
import org.apache.activemq.artemis.api.jms.JMSFactoryType;
import org.apache.activemq.artemis.core.remoting.impl.netty.NettyConnectorFactory;
import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jms.DefaultJmsListenerContainerFactoryConfigurer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Primary;
import org.springframework.jms.annotation.EnableJms;
import org.springframework.jms.config.DefaultJmsListenerContainerFactory;
import org.springframework.jms.config.JmsListenerContainerFactory;
import org.springframework.jms.connection.JmsTransactionManager;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.support.converter.MappingJackson2MessageConverter;
import org.springframework.jms.support.converter.MessageConverter;
import org.springframework.jms.support.converter.MessageType;
#SpringBootApplication
#EnableJms
public class GroupidApplication implements CommandLineRunner {
private static Logger LOG = LoggerFactory
.getLogger(GroupidApplication.class);
#Autowired
private JmsTemplate jmsTemplate;
#Autowired MessageConverter messageConverter;
public static void main(String[] args) {
LOG.info("STARTING THE APPLICATION");
SpringApplication.run(GroupidApplication.class, args);
LOG.info("APPLICATION FINISHED");
}
#Override
public void run(String... args) throws JMSException {
LOG.info("EXECUTING : command line runner");
jmsTemplate.setPubSubDomain(true);
createAndSendTextMessage("Message1");
createAndSendTextMessage("Message2");
createAndSendTextMessage("Message3");
createAndSendTextMessage("Message4");
createAndSendTextMessage("Message5");
createAndSendTextMessage("Message6");
}
private void createAndSendTextMessage(String messageBody) {
jmsTemplate.send("local-queue", session -> {
Message message = session.createTextMessage(messageBody);
message.setStringProperty("JMSXGroupID", "product=paper");
return message;
});
}
// BEANS
#Bean
public JmsListenerContainerFactory<?> containerFactory(ConnectionFactory connectionFactory,
DefaultJmsListenerContainerFactoryConfigurer configurer, JmsTransactionManager jmsTransactionManager) {
DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory();
configurer.configure(factory, connectionFactory);
factory.setSubscriptionDurable(true);
factory.setSubscriptionShared(true);
factory.setSessionAcknowledgeMode(Session.SESSION_TRANSACTED);
factory.setSessionTransacted(Boolean.TRUE);
factory.setTransactionManager(jmsTransactionManager);
return factory;
}
#Bean
public MessageConverter jacksonJmsMessageConverter() {
MappingJackson2MessageConverter converter = new MappingJackson2MessageConverter();
converter.setTargetType(MessageType.TEXT);
converter.setTypeIdPropertyName("_type");
return converter;
}
#Bean
#Primary
public JmsTransactionManager jmsTransactionManager(ConnectionFactory connectionFactory) {
JmsTransactionManager jmsTransactionManager = new JmsTransactionManager(connectionFactory);
// Lazily retrieve existing JMS Connection from given ConnectionFactory
jmsTransactionManager.setLazyResourceRetrieval(true);
return jmsTransactionManager;
}
#Bean
#Primary
public ConnectionFactory connectionFactory() throws JMSException {
// Create ConnectionFactory which enables failover between primary and backup brokers
ActiveMQConnectionFactory activeMqConnectionFactory = ActiveMQJMSClient.createConnectionFactoryWithHA(
JMSFactoryType.CF, transportConfigurations());
activeMqConnectionFactory.setBrokerURL("tcp://localhost:61616?jms.redeliveryPolicy.maximumRedeliveries=1");
activeMqConnectionFactory.setUser("admin");
activeMqConnectionFactory.setPassword("admin");
activeMqConnectionFactory.setInitialConnectAttempts(1);
activeMqConnectionFactory.setReconnectAttempts(5);
activeMqConnectionFactory.setConsumerWindowSize(0);
activeMqConnectionFactory.setBlockOnAcknowledge(true);
activeMqConnectionFactory.setCacheDestinations(true);
activeMqConnectionFactory.setRetryInterval(1000);
return activeMqConnectionFactory;
}
private static TransportConfiguration[] transportConfigurations() {
String connectorFactoryFqcn = NettyConnectorFactory.class.getName();
Map<String, Object> primaryTransportParameters = new HashMap<>(2);
primaryTransportParameters.put("host", "localhost");
primaryTransportParameters.put("port", "61616");
TransportConfiguration primaryTransportConfiguration = new TransportConfiguration(connectorFactoryFqcn,
primaryTransportParameters);
return new TransportConfiguration[] { primaryTransportConfiguration,
new TransportConfiguration(connectorFactoryFqcn) };
}
}
CustomSpringJmsListener:
package com.reproduce.groupid;
import javax.jms.JMSException;
import javax.jms.TextMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.stereotype.Component;
#Component
public class CustomSpringJmsListener {
protected final Logger LOG = LoggerFactory.getLogger(getClass());
#JmsListener(destination = "local-queue", subscription = "groupid-example", containerFactory = "containerFactory", concurrency = "15-15")
public void receive(TextMessage message) throws JMSException {
LOG.info("Received message: " + message);
}
}
pom.xml:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.0.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.reproduce</groupId>
<artifactId>groupid</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>groupid</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-artemis</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
You can see that even though all of these messages have the same group id, they get logged by different listener container threads. If you comment out the transaction manager from the bean definition it starts working again.
It's all about consumer caching. By default, when using an external TXM, caching is disabled so each message is received on a new consumer.
For this app, you really don't need a transaction manager, sessionTransacted is enough - the container will use local transactions.
If you must use an external transaction manager for some reason, consider changing the cache level.
factory.setCacheLevel(DefaultMessageListenerContainer.CACHE_CONSUMER);
See the DMLC javadocs...
/**
* Specify the level of caching that this listener container is allowed to apply.
* <p>Default is {#link #CACHE_NONE} if an external transaction manager has been specified
* (to reobtain all resources freshly within the scope of the external transaction),
* and {#link #CACHE_CONSUMER} otherwise (operating with local JMS resources).
* <p>Some Java EE servers only register their JMS resources with an ongoing XA
* transaction in case of a freshly obtained JMS {#code Connection} and {#code Session},
* which is why this listener container by default does not cache any of those.
* However, depending on the rules of your server with respect to the caching
* of transactional resources, consider switching this setting to at least
* {#link #CACHE_CONNECTION} or {#link #CACHE_SESSION} even in conjunction with an
* external transaction manager.
* #see #CACHE_NONE
* #see #CACHE_CONNECTION
* #see #CACHE_SESSION
* #see #CACHE_CONSUMER
* #see #setCacheLevelName
* #see #setTransactionManager
*/
public void setCacheLevel(int cacheLevel) {

Setting MessageProducer priority has no effect on messages

I have high priority messages that needs to be processed over lower priority messages, but setting the priority on MessageProducer seems to have no impact and the messages are consumed in the same order as they are sent to queue.
Below is my code:
package com.example.jms;
import javax.jms.ConnectionFactory;
import javax.jms.DeliveryMode;
import javax.jms.JMSException;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jms.DefaultJmsListenerContainerFactoryConfigurer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.jms.config.DefaultJmsListenerContainerFactory;
import org.springframework.jms.config.JmsListenerContainerFactory;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.core.ProducerCallback;
import org.springframework.jms.support.converter.MappingJackson2MessageConverter;
import org.springframework.jms.support.converter.MessageConverter;
import org.springframework.jms.support.converter.MessageType;
#SpringBootApplication
public class JmsPriorityApplication {
private static final Logger logger= LoggerFactory.getLogger(JmsPriorityApplication.class);
public static void main(String[] args) {
// Launch the application
ConfigurableApplicationContext context = SpringApplication.run(JmsPriorityApplication.class, args);
JmsTemplate jmsTemplate = context.getBean(JmsTemplate.class);
// Send a message with a POJO - the template reuse the message converter
System.out.println("Sending an email message.");
jmsTemplate.execute("mailbox", new ProducerCallback<Object>() {
#Override
public Object doInJms(Session session, MessageProducer producer) throws JMSException {
String text = "Hello this msg1";
int priority=1;
TextMessage message1 = session.createTextMessage(text);
producer.send(message1, DeliveryMode.PERSISTENT, priority, 0);
logger.info("{} sent with priority={}", text, priority);
text = "Hello this msg2";
priority=9;
TextMessage message2 = session.createTextMessage(text);
producer.send(message2, DeliveryMode.PERSISTENT, priority, 0);
logger.info("{} sent with priority={}", text, priority);
return null;
}
} );
}
#Bean
public JmsListenerContainerFactory<?> myFactory(ConnectionFactory connectionFactory,
DefaultJmsListenerContainerFactoryConfigurer configurer) {
DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory();
// This provides all boot's default to this factory, including the message converter
configurer.configure(factory, connectionFactory);
// You could still override some of Boot's default if necessary.
return factory;
}
}
Receiver.java
package com.example.jms;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.stereotype.Component;
#Component
public class Receiver {
#JmsListener(destination = "mailbox", containerFactory = "myFactory")
public void receiveMessage(String msg) {
System.out.println("Received <" + msg + ">");
}
}
Below is the output:
Sending an email message.
2019-02-05 17:42:44.161 INFO 7828 --- [ main] com.example.jms.JmsPriorityApplication : Hello this msg1 sent with priority=1
2019-02-05 17:42:44.161 INFO 7828 --- [ main] com.example.jms.JmsPriorityApplication : Hello this msg2 sent with priority=9
Received <Hello this msg1>
Received <Hello this msg2>
I was expecting msg2 to be received before msg1. I am not sure what I am missing here. Note: the consumer is active while messages are being sent.
Since the consumer is active when the messages are being sent it's almost certain that the broker is dispatching the first message to the client before the second one arrives on the broker. This means that there's basically no time for the higher priority message to preempt the lower priority message since the client has already received the lower priority message.
In general, the idea of prioritized delivery only makes sense when there is a build-up of messages in the queue. For your test you should either not activate the consumer until all the messages have been sent (a task that would require some kind of external coordination or manual intervention) or increase the message volume so that enough messages build up in the queue the prioritized delivery can actually happen.

Retry mechanism for producer's client of ActiveMQ with JMS and spring

Is there a mechanism or example implementation of a retry mechanism/solution for a producer using ActiveMQ with JMS (more precisely, with JmsTemplate) and spring framework ?
My use case, which I want to handle is, when the broker is not available, for example, I want to make some number of retries, maximum 6 (if possible with exponential delays between each). So, I need also to track the number of retries for a message between each attempt.
I am aware the the redelivery policy for the consumer, but also I want to implement a reliable producer's client side as well
Thanks,
Simeon
i think that the easiest way is to use what exists for this by using an embedded broker with persistence enabled which must be used by the producer to send the messages to and by creating a Camel route to read from local Queue and forward to the remote one or by using a JmsBridgeConnector or NetworkConnector nut i think the JmsBridgeConnector is easier.
here is an Spring code example :
producer have to use jmsConnectionFactory() to create a ConnectionFactory
package com.example.amq;
import java.io.File;
import javax.jms.ConnectionFactory;
import javax.jms.QueueConnectionFactory;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.broker.BrokerService;
import org.apache.activemq.network.jms.JmsConnector;
import org.apache.activemq.network.jms.OutboundQueueBridge;
import org.apache.activemq.network.jms.ReconnectionPolicy;
import org.apache.activemq.network.jms.SimpleJmsQueueConnector;
import org.apache.activemq.store.PersistenceAdapter;
import org.apache.activemq.store.kahadb.KahaDBPersistenceAdapter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
#Configuration
public class ActiveMQConfiguration {
public static final String DESTINATION_NAME = "localQ";
#Bean // (initMethod = "start", destroyMethod = "stop")
public BrokerService broker() throws Exception {
final BrokerService broker = new BrokerService();
broker.addConnector("vm://localhost");
SimpleJmsQueueConnector simpleJmsQueueConnector = new SimpleJmsQueueConnector();
OutboundQueueBridge bridge = new OutboundQueueBridge();
bridge.setLocalQueueName(DESTINATION_NAME);
bridge.setOutboundQueueName("remoteQ");
OutboundQueueBridge[] outboundQueueBridges = new OutboundQueueBridge[] { bridge };
simpleJmsQueueConnector.getReconnectionPolicy().setMaxSendRetries(ReconnectionPolicy.INFINITE);
simpleJmsQueueConnector.setOutboundQueueBridges(outboundQueueBridges);
simpleJmsQueueConnector.setLocalQueueConnectionFactory((QueueConnectionFactory) jmsConnectionFactory());
simpleJmsQueueConnector.setOutboundQueueConnectionFactory(outboundQueueConnectionFactory());
JmsConnector[] jmsConnectors = new JmsConnector[] { simpleJmsQueueConnector };
broker.setJmsBridgeConnectors(jmsConnectors);
PersistenceAdapter persistenceAdapter = new KahaDBPersistenceAdapter();
File dir = new File(System.getProperty("user.home") + File.separator + "kaha");
if (!dir.exists()) {
dir.mkdirs();
}
persistenceAdapter.setDirectory(dir);
broker.setPersistenceAdapter(persistenceAdapter);
broker.setPersistent(true);
broker.setUseShutdownHook(false);
broker.setUseJmx(true);
return broker;
}
#Bean
public QueueConnectionFactory outboundQueueConnectionFactory() {
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory(
"auto://localhost:5671");
connectionFactory.setUserName("admin");
connectionFactory.setPassword("admin");
return connectionFactory;
}
#Bean
public ConnectionFactory jmsConnectionFactory() {
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("vm://localhost");
connectionFactory.setObjectMessageSerializationDefered(true);
connectionFactory.setCopyMessageOnSend(false);
return connectionFactory;
}
}
By using Camel :
import org.apache.activemq.camel.component.ActiveMQComponent;
import org.apache.activemq.camel.component.ActiveMQConfiguration;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
public class ActiveMQCamelBridge {
public static void main(String args[]) throws Exception {
CamelContext context = new DefaultCamelContext();
context.addComponent("inboundQueue", ActiveMQComponent.activeMQComponent("tcp://localhost:61616"));
ActiveMQComponent answer = ActiveMQComponent.activeMQComponent("tcp://localhost:5671");
if (answer.getConfiguration() instanceof ActiveMQConfiguration) {
((ActiveMQConfiguration) answer.getConfiguration()).setUserName("admin");
((ActiveMQConfiguration) answer.getConfiguration()).setPassword("admin");
}
context.addComponent("outboundQueue", answer);
context.addRoutes(new RouteBuilder() {
public void configure() {
from("inboundQueue:queue:localQ").to("outboundQueue:queue:remoteQ");
}
});
context.start();
Thread.sleep(60 * 5 * 1000);
context.stop();
}
}
Producer does not provide any kind of retry mechanism like consumer. You need to make sure in your code that message sent by producer acknowledge by broker.

Spring Boot with Apache Kafka: Messages not being read

I am currently setting up a Spring Boot application with Kafka listener.
I am trying to code only the consumer. For producer, I am manually sending message from the Kafka console for now.
I followed the example:
http://www.source4code.info/2016/09/spring-kafka-consumer-producer-example.html
I tried running this as a Spring Boot application but not able to see any messages being received. There are already some messages in my local topic of Kafka.
C:\software\kafka_2.11-0.10.1.0\kafka_2.11-0.10.1.0\kafka_2.11-0.10.1.0\bin\wind
ows>kafka-console-producer.bat --broker-list localhost:9092 --topic test
this is a message
testing again
My Spring Boot application is:
#EnableDiscoveryClient
#SpringBootApplication
public class KafkaApplication {
/**
* Run the application using Spring Boot and an embedded servlet engine.
*
* #param args
* Program arguments - ignored.
*/
public static void main(String[] args) {
// Tell server to look for registration.properties or registration.yml
System.setProperty("spring.config.name", "kafka-server");
SpringApplication.run(KafkaApplication.class, args);
}
}
And Kafka configuration is:
package kafka;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.IntegerDeserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import java.util.HashMap;
import java.util.Map;
#Configuration
#EnableKafka
public class KafkaConsumerConfig {
#Bean
KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory();
factory.setConsumerFactory(consumerFactory());
//factory.setConcurrency(1);
//factory.getContainerProperties().setPollTimeout(3000);
return factory;
}
#Bean
public ConsumerFactory<String, String> consumerFactory() {
return new DefaultKafkaConsumerFactory(consumerConfigs());
}
#Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> propsMap = new HashMap();
propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
//propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
//propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
//propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
//propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, "group1");
//propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
return propsMap;
}
#Bean
public Listener listener() {
return new Listener();
}
}
And Kafka listener is:
package kafka;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.logging.Logger;
public class Listener {
protected Logger logger = Logger.getLogger(Listener.class
.getName());
public CountDownLatch getCountDownLatch1() {
return countDownLatch1;
}
private CountDownLatch countDownLatch1 = new CountDownLatch(1);
#KafkaListener(topics = "test")
public void listen(ConsumerRecord<?, ?> record) {
logger.info("Received message: " + record);
System.out.println("Received message: " + record);
countDownLatch1.countDown();
}
}
I am trying this for the first time. Please let me know if I am doing anything wrong. Any help will be greatly appreciated.
You did not set ConsumerConfig.AUTO_OFFSET_RESET_CONFIG so the default is "latest". Set it to "earliest" so the consumer will receive messages already in the topic.
ConsumerConfig.AUTO_OFFSET_RESET_CONFIG takes effect only if the consumer group does not already have an offset for a topic partition. If you already ran the consumer with the "latest" setting, then running the consumer again with a different setting does not change the offset. The consumer must use a different group so Kafka will assign offsets for that group.
Observed that you dit comment out the consumer group.id property.
//propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, "group1");
Let's see how is quoted in the Kafka official document:
A unique string that identifies the consumer group this consumer belongs to. This property is required if the consumer uses either the group management functionality by using subscribe(topic) or the Kafka-based offset management strategy.
Tried to uncomement that row and the consumer worked.
You will need to annotate your Listener class with either #Service or #Component so that Spring Boot can load the Kafka listener.
package kafka;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.logging.Logger;
#Component
public class Listener {
protected Logger logger = Logger.getLogger(Listener.class
.getName());
public CountDownLatch getCountDownLatch1() {
return countDownLatch1;
}
private CountDownLatch countDownLatch1 = new CountDownLatch(1);
#KafkaListener(topics = "test")
public void listen(ConsumerRecord<?, ?> record) {
logger.info("Received message: " + record);
System.out.println("Received message: " + record);
countDownLatch1.countDown();
}
}
The above suggestions are good. If you have followed all of them but it did not work, please check if lazy loading is set to false for your application.
The lazy loading is false by default. However if your application had explicit setting like the one below,
spring.main.lazy-initialization=true
Please comment it or make it to false

How to listen to topic using spring boot jms

I am trying to listen to topic using the below snippet. However its listening to queue by default. There is no xml config in this case. I am completely relying on annotations. Moreover I have relied completely on the AutoConfiguration provided by Spring boot. I am not sure how to set the destination type as topic, In JmsListener. Spring JMS gurus please help.
#Component
public class MyTopicListener {
#JmsListener(destination = "${trans.alert.topic}")
public void receiveMessage(TransactionAlert alert) {
logger.info("AlertSubscriberEmail :: Sending Email => <" + alert + ">");
}
}
The answer marked correct is ALMOST correct. It still wont work because:
factory.setPubSubDomain(true)
must come AFTER:
configurer.configure(factory, connectionFactory);
Otherwise the pubSubDomain flag being set to true is lost when configuring the defaults and that factory instance will still work with queues and not topics.
I just took the complete Spring boot example from : https://github.com/spring-guides/gs-messaging-jms/
In this it is created for sending and receipt of messages from a queue. To Change this to a topic , you have to set the Pub-Sub property in the Factory instance.
import org.springframework.beans.factory.support.BeanDefinitionBuilder;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jms.DefaultJmsListenerContainerFactoryConfigurer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.jms.annotation.EnableJms;
import org.springframework.jms.config.DefaultJmsListenerContainerFactory;
import org.springframework.jms.config.JmsListenerContainerFactory;
import org.springframework.jms.connection.CachingConnectionFactory;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.support.converter.MappingJackson2MessageConverter;
import org.springframework.jms.support.converter.MessageConverter;
import org.springframework.jms.support.converter.MessageType;
import javax.jms.ConnectionFactory;
#SpringBootApplication
#EnableJms
public class JmsSampleApplication {
public void registerBeans(ConfigurableApplicationContext context ){
BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(JmsTemplate.class);
CachingConnectionFactory cachingConnectionFactory = new CachingConnectionFactory();
builder.addPropertyValue("connectionFactory", cachingConnectionFactory); // set property value
DefaultListableBeanFactory factory = (DefaultListableBeanFactory) context.getAutowireCapableBeanFactory();
factory.registerBeanDefinition("jmsTemplateName", builder.getBeanDefinition());
}
#Bean
public JmsListenerContainerFactory<?> topicListenerFactory(ConnectionFactory connectionFactory,
DefaultJmsListenerContainerFactoryConfigurer configurer) {
DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory();
factory.setPubSubDomain(true);
// This provides all boot's default to this factory, including the message converter
configurer.configure(factory, connectionFactory);
// You could still override some of Boot's default if necessary.
return factory;
}
#Bean
public JmsListenerContainerFactory<?> queueListenerFactory(ConnectionFactory connectionFactory,
DefaultJmsListenerContainerFactoryConfigurer configurer) {
DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory();
//factory.setPubSubDomain(true);
// This provides all boot's default to this factory, including the message converter
configurer.configure(factory, connectionFactory);
return factory;
}
#Bean // Serialize message content to json using TextMessage
public MessageConverter jacksonJmsMessageConverter() {
MappingJackson2MessageConverter converter = new MappingJackson2MessageConverter();
converter.setTargetType(MessageType.TEXT);
converter.setTypeIdPropertyName("_type");
return converter;
}
public static void main(String[] args) {
ConfigurableApplicationContext context = SpringApplication.run(JmsSampleApplication.class, args);
JmsTemplate jmsTemplate = context.getBean(JmsTemplate.class);
// Send a message with a POJO - the template reuse the message converter
System.out.println("Sending an email message.");
jmsTemplate.convertAndSend("mailbox.topic", new Email("info#example.com", "Hello"));
jmsTemplate.convertAndSend("mailbox.queue", new Email("info#example.com", "Hello"));
}
}
The listener
package org.springboot.jms;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.stereotype.Component;
/**
* Created by RGOVIND on 10/20/2016.
*/
#Component
public class HelloTopicListener {
#JmsListener(destination = "mailbox.topic", containerFactory = "topicListenerFactory")
public void receiveTopicMessage(Email email) {
System.out.println("Received <" + email + ">");
}
#JmsListener(destination = "mailbox.queue", containerFactory = "queueListenerFactory")
public void receiveQueueMessage(Email email) {
System.out.println("Received <" + email + ">");
}
}
Once this is done , you are all set to subscribe to the topic of choice.
There are multiple approaches to this of course , you can have a map of beans for different jmsTemplates , each of which can be used when you need them based on queue or topic. The template & beans can be instantiated in a method you choose to like discussed in this SO Question. Hope it helps
In Spring Boot's Application.properties, try setting the following property:
spring.jms.pub-sub-domain=true
Then, use this property for the container factory that you are using to listen to the topic.

Resources