I need to set Transformation( String s) before enqueue to Oracle AQ. I have send details using JmsTemplate.where I am not able to see any method to do so
private static final String QUEUENAME_WRITE = "SA.CLFY_EVENT_Q";
private AQEnqueueOptions aqEnqueueOptions;
#Bean
/**
* Spring bean to WRITE/SEND/ENQUEUE messages on a queue with a certain name
*/
public JmsTemplate jmsTemplate(#Qualifier("clifyConnectionFactory")ConnectionFactory conFactory) {
JmsTemplate jmsTemplate = new JmsTemplate();
jmsTemplate.setDefaultDestinationName(QUEUENAME_WRITE);
jmsTemplate.setSessionTransacted(true);
jmsTemplate.setConnectionFactory(conFactory);
jmsTemplate.setPubSubDomain(true);
jmsTemplate.setMessageConverter(new MappingAdtMessageConverter(new EventMapper()));
return jmsTemplate;
}
Related
I am using spring boot 2.1.9 with spring Kafka 2.2.9.
If the message failed a number of times (defined in the afterRollbackProcessor), The consumer stops polling the record. but if the consumer restarted, it again re-poll the same message and processes.
But I don't want the messages to be re-polled again, What is the best way to stop it?
here is my config
#Configuration
#EnableKafka
public class KafkaReceiverConfig {
// Kafka Server Configuration
#Value("${kafka.servers}")
private String kafkaServers;
// Group Identifier
#Value("${kafka.groupId}")
private String groupId;
// Kafka Max Retry Attempts
#Value("${kafka.retry.maxAttempts:5}")
private Integer retryMaxAttempts;
// Kafka Max Retry Interval
#Value("${kafka.retry.interval:180000}")
private Long retryInterval;
// Kafka Concurrency
#Value("${kafka.concurrency:10}")
private Integer concurrency;
// Kafka Concurrency
#Value("${kafka.poll.timeout:300}")
private Integer pollTimeout;
// Kafka Consumer Offset
#Value("${kafka.consumer.auto-offset-reset:earliest}")
private String offset = "earliest";
#Value("${kafka.max.records:100}")
private Integer maxPollRecords;
#Value("${kafka.max.poll.interval.time:500000}")
private Integer maxPollIntervalMs;
#Value("${kafka.max.session.timeout:60000}")
private Integer sessionTimoutMs;
// Logger
private static final Logger log = LoggerFactory.getLogger(KafkaReceiverConfig.class);
#Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory(
ChainedKafkaTransactionManager<String, String> chainedTM, MessageProducer messageProducer) {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(concurrency);
factory.getContainerProperties().setPollTimeout(pollTimeout);
factory.getContainerProperties().setAckMode(AckMode.RECORD);
factory.getContainerProperties().setSyncCommits(true);
factory.getContainerProperties().setAckOnError(false);
factory.getContainerProperties().setTransactionManager(chainedTM);
AfterRollbackProcessor<String, String> afterRollbackProcessor = new DefaultAfterRollbackProcessor<>(
(record, exception) -> {
log.warn("failed to process kafka message (retries are exausted). topic name:" + record.topic()
+ " value:" + record.value());
messageProducer.saveFailedMessage(record, exception);
}, retryMaxAttempts);
factory.setAfterRollbackProcessor(afterRollbackProcessor);
log.debug("Kafka Receiver Config kafkaListenerContainerFactory created");
return factory;
}
#Bean
public ConsumerFactory<String, String> consumerFactory() {
log.debug("Kafka Receiver Config consumerFactory created");
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
#Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new ConcurrentHashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServers);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs);
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimoutMs);
props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, offset);
props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed");
log.debug("Kafka Receiver Config consumerConfigs created");
return props;
}
}
How can I achieve this?
Set the commitRecovered property to true and inject a KafkaTemplate configured with the same producer factory as the transaction manager.
/**
* {#inheritDoc}
* Set to true and the container will run the
* {#link #process(List, Consumer, Exception, boolean)} method in a transaction and,
* if a record is skipped and recovered, we will send its offset to the transaction.
* Requires a {#link KafkaTemplate}.
* #param commitRecovered true to process in a transaction.
* #since 2.2.5
* #see #isProcessInTransaction()
* #see #process(List, Consumer, Exception, boolean)
* #see #setKafkaTemplate(KafkaTemplate)
*/
#Override
public void setCommitRecovered(boolean commitRecovered) { // NOSONAR enhanced javadoc
super.setCommitRecovered(commitRecovered);
}
EDIT
Here's the logic in process...
if (SeekUtils.doSeeks(((List) records), consumer, exception, recoverable,
getSkipPredicate((List) records, exception), this.logger)
&& isCommitRecovered() && this.kafkaTemplate != null && this.kafkaTemplate.isTransactional()) {
// if we get here it means retries are exhausted and we've skipped
ConsumerRecord<K, V> skipped = records.get(0);
this.kafkaTemplate.sendOffsetsToTransaction(
Collections.singletonMap(new TopicPartition(skipped.topic(), skipped.partition()),
new OffsetAndMetadata(skipped.offset() + 1)));
}
EDIT2
In 2.2.x, the property is
/**
* Set to true to run the {#link #process(List, Consumer, Exception, boolean)}
* method in a transaction. Requires a {#link KafkaTemplate}.
* #param processInTransaction true to process in a transaction.
* #since 2.2.5
* #see #process(List, Consumer, Exception, boolean)
* #see #setKafkaTemplate(KafkaTemplate)
*/
public void setProcessInTransaction(boolean processInTransaction) {
this.processInTransaction = processInTransaction;
}
I am using a spring boot 2.1.9 with Kafka and MySQL and also implemented a chained transaction manager.
I want to set the backOffPolicy so that the retry can happen after a certain time. it's possible in the new spring Kafka version but due to some other dependencies, I could not able to upgrade spring boot.
As of now, I am using AfterRollbackProcessor to handle failed messages, now I want to implement backoffPolicy with AfterRollbackProcessor using Spring Kafka 2.2.9.RELEASE. Is there any way to implement it?
here is reciever config file:
#Configuration
#EnableKafka
public class KafkaReceiverConfig {
// Kafka Server Configuration
#Value("${kafka.servers}")
private String kafkaServers;
// Group Identifier
#Value("${kafka.groupId}")
private String groupId;
// Kafka Max Retry Attempts
#Value("${kafka.retry.maxAttempts:5}")
private Integer retryMaxAttempts;
// Kafka Max Retry Interval
#Value("${kafka.retry.interval:180000}")
private Long retryInterval;
// Kafka Concurrency
#Value("${kafka.concurrency:10}")
private Integer concurrency;
// Kafka Concurrency
#Value("${kafka.poll.timeout:300}")
private Integer pollTimeout;
// Kafka Consumer Offset
#Value("${kafka.consumer.auto-offset-reset:earliest}")
private String offset = "earliest";
#Value("${kafka.max.records:100}")
private Integer maxPollRecords;
#Value("${kafka.max.poll.interval.time:500000}")
private Integer maxPollIntervalMs;
#Value("${kafka.max.session.timeout:60000}")
private Integer sessionTimoutMs;
// Logger
private static final Logger log = LoggerFactory.getLogger(KafkaReceiverConfig.class);
/**
* String Kafka Listener Container Factor
*
* #return #see {#link KafkaListenerContainerFactory}
*/
#Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory(
ChainedKafkaTransactionManager<String, String> chainedTM, MessageProducer messageProducer) {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setConcurrency(concurrency);
factory.getContainerProperties().setPollTimeout(pollTimeout);
factory.getContainerProperties().setAckMode(AckMode.RECORD);
factory.getContainerProperties().setSyncCommits(true);
// factory.setRetryTemplate(retryTemplate());
factory.getContainerProperties().setAckOnError(false);
factory.getContainerProperties().setTransactionManager(chainedTM);
// factory.setStatefulRetry(true);
AfterRollbackProcessor<String, String> afterRollbackProcessor = new DefaultAfterRollbackProcessor<>(
(record, exception) -> {
log.warn("failed to process kafka message (retries are exausted). topic name:" + record.topic()
+ " value:" + record.value());
messageProducer.saveFailedMessage(record, exception);
}, retryMaxAttempts);
factory.setAfterRollbackProcessor(afterRollbackProcessor);
log.debug("Kafka Receiver Config kafkaListenerContainerFactory created");
return factory;
}
/**
* String Consumer Factory
*
* #return #see {#link ConsumerFactory}
*/
#Bean
public ConsumerFactory<String, String> consumerFactory() {
log.debug("Kafka Receiver Config consumerFactory created");
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
/**
* Consumer Configurations
*
* #return #see {#link Map}
*/
#Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new ConcurrentHashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServers);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs);
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimoutMs);
props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, offset);
props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed");
log.debug("Kafka Receiver Config consumerConfigs created");
return props;
}
}
You can use listener retry but it MUST be stateful (you have that commented out). Otherwise, the retries will be performed within the transaction which is generally not what you want.
With stateful retry, the template throws the exception after it backs off; then the after rollback processor will perform a re-seek so the record is reprocessed.
As you say, in 2.3 we added a BackOff to the after rollback processor to make it easier to configure everything all in one place.
I'm trying to attach a custom message converter that implements org.springframework.jms.support.converter.MessageConverter, to a JmsMessagingTemplate.
I've read somewhere that we can attach the message converter to a MessagingMessageConverter by calling setPayloadConverter, and then attach that messaging message converter to the JmsMessagingTemplate via setJmsMessageConverter. After that, I call convertAndSend, but I notice that it doesn't convert the payload.
When I debugged the code, I notice that setting Jms Message Converter doesn't set the converter instance variable in the JmsMessagingTemplate. So when the convertAndSend method calls doConvert and tries to getConverter, it is getting the default simple message converter and not my custom one.
My question is, can I use an implementation of org.springframework.jms.support.converter.MessageConverter with a JmsMessagingTemplate? Or do I need to use an implementation of org.springframework.messaging.converter.MessageConverter?
I'm using Spring Boot 1.4.1.RELEASE, and Spring 4.3.3.RELEASE. The code is below.
Configuration
#Configuration
#EnableJms
public class MessagingEncryptionPocConfig {
/**
* Listener ActiveMQ Connection Factory
*/
#Bean(name="listenerActiveMqConnectionFactory")
public ActiveMQConnectionFactory listenerActiveMqConnectionFactory() {
return new ActiveMQConnectionFactory("admin","admin","tcp://localhost:61616");
}
/**
* Producer ActiveMQ Connection Factory
*/
#Bean(name="producerActiveMqConnectionFactory")
public ActiveMQConnectionFactory producerActiveMqConnectionFactory() {
return new ActiveMQConnectionFactory("admin","admin","tcp://localhost:61616");
}
/**
* Caching Connection Factory
*/
#Bean
public CachingConnectionFactory cachingConnectionFactory(#Qualifier("producerActiveMqConnectionFactory") ActiveMQConnectionFactory activeMqConnectionFactory) {
return new CachingConnectionFactory(activeMqConnectionFactory);
}
/**
* JMS Listener Container Factory
*/
#Bean
public DefaultJmsListenerContainerFactory jmsListenerContainerFactory(#Qualifier("listenerActiveMqConnectionFactory") ActiveMQConnectionFactory connectionFactory, MessagingMessageConverter messageConverter) {
DefaultJmsListenerContainerFactory defaultJmsListenerContainerFactory = new DefaultJmsListenerContainerFactory();
defaultJmsListenerContainerFactory.setConnectionFactory(connectionFactory);
defaultJmsListenerContainerFactory.setMessageConverter(messageConverter);
return defaultJmsListenerContainerFactory;
}
/**
* Jms Queue Template
*/
#Bean(name="queueTemplate")
public JmsMessagingTemplate queueTemplate(CachingConnectionFactory cachingConnectionFactory, MessageConverter messagingMessageConverter) {
JmsMessagingTemplate queueTemplate = new JmsMessagingTemplate(cachingConnectionFactory);
queueTemplate.setJmsMessageConverter(messagingMessageConverter);
return queueTemplate;
}
#Bean
public MessageConverter encryptionDecryptionMessagingConverter(Jaxb2Marshaller jaxb2Marshaller) {
MessageConverter encryptionDecryptionMessagingConverter = new EncryptionDecryptionMessagingConverter(jaxb2Marshaller);
MessagingMessageConverter messageConverter = new MessagingMessageConverter();
messageConverter.setPayloadConverter(encryptionDecryptionMessagingConverter);
return messageConverter;
}
/**
* Jaxb marshaller
*/
#Bean(name="producerJaxb2Marshaller")
public Jaxb2Marshaller jaxb2Marshaller() {
Jaxb2Marshaller jaxb2Marshaller = new Jaxb2Marshaller();
jaxb2Marshaller.setPackagesToScan("com.schema");
return jaxb2Marshaller;
}
}
MessageProducer Class
#Component
public class MessageProducer {
private static final Logger LOG = LoggerFactory.getLogger(MessageProducer.class);
#Autowired
#Qualifier("queueTemplate")
private JmsMessagingTemplate queueTemplate;
public void publishMsg(Transaction trx, Map<String,Object> jmsHeaders, MessagePostProcessor postProcessor) {
LOG.info("Sending Message. Payload={} Headers={}",trx,jmsHeaders);
queueTemplate.convertAndSend("queue.source", trx, jmsHeaders, postProcessor);
}
}
Unit Test
#RunWith(SpringRunner.class)
#SpringBootTest
#ActiveProfiles("test")
public class WebsMessagingEncryptionPocApplicationTests {
#Autowired
private MessageProducer producer;
#Autowired
private MessageListener messageListener;
/**
* Ensure that a message is sent, and received.
*/
#Test
public void testProducer() throws Exception{
//ARRANGE
CountDownLatch latch = new CountDownLatch(1);
messageListener.setCountDownLatch(latch);
Transaction trx = new Transaction();
trx.setCustomerAccountID(new BigInteger("111111"));
Map<String,Object> jmsHeaders = new HashMap<String,Object>();
jmsHeaders.put("tid", "1234563423");
MessagePostProcessor encryptPostProcessor = new EncryptMessagePostProcessor();
//ACT
producer.publishMsg(trx, jmsHeaders, encryptPostProcessor);
latch.await();
//ASSERT - assertion done in the consumer
}
}
The converter field is used to convert your input params to a spring-messaging Message<?>.
The JMS converter is used later (in MessagingMessageCreator) to then create a JMS Message from the messaging Message<?>.
I have a consumer consumes a message do some transformation and create a new Pojo and pass to a producer.
The producer sends the message in queue using the JmsTemplate.
The producer should set the headers of the original message like (JMSType, JMSCorrelationID, JMSExpiration, JMSDeliveryMode) to the new message to send.
But the producer should to change the replyTo Destination of the original message.
I haven't found a way to create a Destination and set to the JMSReplyTo.
Some have an idea how I can do that?
Maybe the JmsTemplate is not the correct class to do that.
public class Producer {
private final JmsTemplate jmsTemplate;
#Value("${jms-destination}")
private String destination;
public Producer(#Autowired JmsTemplate jmsTemplate) {
this.jmsTemplate = jmsTemplate;
}
public void send(final MessageHeaders headers, final Pojo pojo) {
Validate.notNull(order);
jmsTemplate.convertAndSend(destination, pojo, (message) -> {
final Destination replyToDestination = ???;
message.setJMSReplyTo(replyToDestination);
message.setJMSType((String) headers.get(JmsHeaders.TYPE));
message.setJMSCorrelationID((String) headers.get(JmsHeaders.CORRELATION_ID));
message.setJMSExpiration((long) headers.get(JmsHeaders.EXPIRATION));
message.setJMSDeliveryMode((int) headers.get(JmsHeaders.DELIVERY_MODE));
return message;
});
}
}
I have found only this way to do, but I don't like and I don't sure that this introduce side effect:
public class Producer {
private final JmsTemplate jmsTemplate;
#Value("${jms-destination}")
private String destination;
#Value("${jms-replyTo}")
private String replyTo;
public Producer(#Autowired JmsTemplate jmsTemplate) {
this.jmsTemplate = jmsTemplate;
}
public void send(final MessageHeaders headers, Pojo pojo) {
Validate.notNull(order);
jmsTemplate.convertAndSend(destination, order, (message) -> {
final Destination replyToDestination = buildReplyTo();
message.setJMSReplyTo(replyToDestination);
message.setJMSType((String) headers.get(JmsHeaders.TYPE));
message.setJMSCorrelationID((String) headers.get(JmsHeaders.CORRELATION_ID));
message.setJMSExpiration((long) headers.get(JmsHeaders.EXPIRATION));
message.setJMSDeliveryMode((int) headers.get(JmsHeaders.DELIVERY_MODE));
return message;
});
}
private Destination buildReplyTo() throws JMSException {
final Session session = jmsTemplate.getConnectionFactory().createConnection()
.createSession(false, Session.AUTO_ACKNOWLEDGE);
final Destination queue =
jmsTemplate.getDestinationResolver().resolveDestinationName(session, replyTo, false);
return queue;
}
}
Your solution creates side-along connections, which are not closed.
You should use existing session object and send your pojo manually through send API. Use getRequiredMessageConverter to convert your pojo.
public void send(final MessageHeaders headers, Pojo pojo) {
Validate.notNull(order);
final String responseQueue = "responseQ";
jmsTemplate.send(destination,
session -> {
Message message = getRequiredMessageConverter().toMessage(message, session);
message.setJMSReplyTo(session.createQueue(responseQueue)); //fill queue
//any other setters
return message;
});
}
// based on Spring JmsTemplate library
private MessageConverter getRequiredMessageConverter() throws IllegalStateException {
MessageConverter converter = jmsTemplate.getMessageConverter();
if (converter == null) {
throw new IllegalStateException("No 'messageConverter' specified. Check configuration of JmsTemplate.");
} else {
return converter;
}
}
I'm using spring integration to poll a database using a JdbcPollingChannelAdapter and then post the results onto an Activemq queue using JmsSendingMessageHandler. I'm serializing the jdbc results as json string using a MappingJackson2MessageConverter. When the message get's sent, it gets sent as an arraylist. Is it possible to only send a single json-serialized object with the payload of a message at a time? This would allow me to then listen onto the queue like so
#JmsListener(destination = "${activemq.queue.name}")
public void receive(DomainObj obj)
Spring Integration Configuration
#Configuration
public class SpringIntegrationConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(SpringIntegrationConfig.class);
#Value("${database.polling-interval.rate-in-milliseconds}")
private Long pollingRateInMilliSeconds;
#Value("${database.max-messages-per-poll}")
private Long maxMessagesPerPoll;
#Bean
public MessageChannel helloWorldChannel() {
return new DirectChannel();
}
#Bean
public PollerMetadata poller(PlatformTransactionManager transactionManager) {
PeriodicTrigger trigger = new PeriodicTrigger(pollingRateInMilliSeconds);
trigger.setFixedRate(true);
MatchAlwaysTransactionAttributeSource attributeSource = new MatchAlwaysTransactionAttributeSource();
attributeSource.setTransactionAttribute(new DefaultTransactionAttribute());
TransactionInterceptor interceptor = new TransactionInterceptor(transactionManager, attributeSource);
PollerMetadata poller = new PollerMetadata();
poller.setTrigger(trigger);
poller.setMaxMessagesPerPoll(maxMessagesPerPoll);
poller.setAdviceChain(Collections.singletonList(interceptor));
return poller;
}
#Bean
#InboundChannelAdapter(value = "helloWorldChannel", channel = "helloWorldChannel", poller = #Poller("poller"))
public MessageSource<?> helloWorldMessageSource(DataSource dataSource) {
JdbcPollingChannelAdapter adapter = new JdbcPollingChannelAdapter(dataSource, "select * from item where type = 2");
adapter.setUpdateSql("update item set type = 10 where id in (:id)");
adapter.setRowMapper(new ItemRowMapper());
adapter.setMaxRowsPerPoll(maxMessagesPerPoll.intValue());
return adapter;
}
#Bean
#ServiceActivator(inputChannel = "helloWorldChannel")
public MessageHandler jsmOutboundAdapter(JmsTemplate template, Queue queue, MessageConverter converter) {
template.setMessageConverter(converter);
JmsSendingMessageHandler handler = new JmsSendingMessageHandler(template);
handler.setDestination(queue);
return handler;
}
#Bean // Serialize message content to json using TextMessage
public MessageConverter jsonJmsMessageConverter() {
MappingJackson2MessageConverter converter = new MappingJackson2MessageConverter();
converter.setTargetType(MessageType.TEXT);
converter.setTypeIdPropertyName("_type");
return converter;
}
}
Change your select statement to use the JDBC vendor syntax to only retrieve one record - e.g. LIMIT 1.
Then, remove the setMaxRowsPerPoll() (leave it to default at 0) and you will get the single result.
#Transformer public Object transform(List<Object> list) { return list.get(0); } worked with the use of a SQL LIMIT. That's how you get it to return a json object, instead of an array.