Using Spring Integration for transferring message from one queue to other in Websphere MQ.
OutPut Channel shows message is going but I am not able to see in Queue from MQ Explorer.How the session commits ? and I am not able to stop the java application. How to gracefully stop with connection stop ?
where I am doing wrong ?
Here is the context config -
<int:logging-channel-adapter log-full-message="true" id="logger" level="INFO"/>
<bean id="jmsConnectionFactory" class="com.ibm.mq.jms.MQQueueConnectionFactory">
<property name="hostName" value="localhost" />
<property name="port" value="1414" />
<property name="queueManager" value="TEST" />
<property name="transportType" value="1" />
</bean>
<bean id="inQueue" class="com.ibm.mq.jms.MQQueue">
<constructor-arg value="TESTQ" />
</bean>
<bean id="outQueue" class="com.ibm.mq.jms.MQQueue">
<constructor-arg value="DEST_QUEUE" />
</bean>
<int:channel id="readFromChannel">
<int:interceptors>
<int:wire-tap channel="logger"/>
</int:interceptors>
</int:channel>
<int:channel id="sendToChannel" >
<int:queue/>
<int:interceptors>
<int:wire-tap channel="logger"/>
</int:interceptors>
</int:channel>
<int-jms:message-driven-channel-adapter id="jmsInAdapter"
connection-factory="cachingConnectionFactory"
destination="inQueue"
channel="readFromChannel" />
<int-jms:message-driven-channel-adapter id="jmsOutAdapter"
connection-factory="cachingConnectionFactory"
destination="outQueue"
channel="sendToChannel" />
<bean id="myMessenger" class="test.MyMessenger" />
<int:service-activator id="servAct" input-channel="readFromChannel" output-channel="sendToChannel" ref="myMessenger" method="transfer"/>
If I understand your use-case correctly, you want to shift messages from TESTQ to DEST_QUEUE. But you have <int-jms:message-driven-channel-adapter> for both to read messages from them, but no one to put messages to DEST_QUEUE.
Your <int:service-activator> get message from readFromChannel and it means 'get message from TESTQ', but it places message to the sendToChannel. But it doesn't say that you place a message to the DEST_QUEUE.
That is because <int-jms:message-driven-channel-adapter> is for reading messages from JMS (MQ in your case).
To send message to JMS (to the DEST_QUEUE in your case) you have to use <int-jms:outbound-channel-adapter> and there is no reason to mark sendToChannel as <queue>
Related
I'm using
Spring Batch
Step 1
Step 2 Master (Partitioner)
Step 3
Spring Integration (JMS) to communicate Master and Slave
The issue we are seeing is, the first slave handles all JMS messages instead of even distribution between slaves.
See configuration as below
Master
<bean id="PreProcess" class="com.job.tasklet.PreProcessTasklet" scope="step">
<constructor-arg index="0" value="${run.slave}"/>
<property name="maxNumberOfSlaves" value="#{jobParameters['max-slave-count']}"/>
</bean>
<bean id="PostProcess" class="com.job.tasklet.PostProcessTasklet" scope="prototype">
<constructor-arg index="0" ref="chpsJobDataSource"/>
</bean>
<bean id="partitioner" class="com.job.partition.DatabasePartitioner" scope="step">
<constructor-arg index="3" value="${max.row.count}"/>
</bean>
<bean id="partitionHandler" class="com.job.handler.StepExecutionAggregatorHandler">
<property name="stepName" value="processAutoHoldSlaveStep"/>
<property name="gridSize" value="${grid.size}"/>
<property name="replyChannel" ref="aggregatedGroupRuleReplyChannel"/>
<property name="messagingOperations">
<bean class="org.springframework.integration.core.MessagingTemplate">
<property name="defaultChannel" ref="groupRuleRequestsChannel"/>
</bean>
</property>
</bean>
<!-- Request Start -->
<int:channel id="groupRuleRequestsChannel" />
<int-jms:outbound-channel-adapter channel="groupRuleRequestsChannel" jms-template="jmsTemplateToSlave"/>
<bean id="jmsTemplateToSlave" class="org.springframework.jms.core.JmsTemplate">
<property name="connectionFactory" ref="connectionFactory"/>
<property name="receiveTimeout" value="5000"/>
<property name="defaultDestinationName" value="defaultRequest"/>
</bean>
<bean id="jmsTemplateFromSlave" class="org.springframework.jms.core.JmsTemplate" parent="jmsTemplateToSlave">
<property name="defaultDestinationName" value="defaultRequest"/>
</bean>
<!-- Response Test Start -->
<int:channel id="groupRuleReplyChannel">
<!-- <int:queue/> -->
</int:channel>
<int-jms:inbound-channel-adapter channel="groupRuleReplyChannel" jms-template="jmsTemplateFromSlave">
<int:poller id="defaultPoller" default="true" max-messages-per-poll="1" fixed-rate="3000" />
</int-jms:inbound-channel-adapter>
<!-- define aggregatedReplyChannel -->
<int:channel id="aggregatedGroupRuleReplyChannel">
<int:queue/>
</int:channel>
<int:aggregator ref="partitionHandler"
input-channel="groupRuleReplyChannel"
output-channel="aggregatedGroupRuleReplyChannel"
send-timeout="3600000"/>
Slave
<int:channel id="requestsChannel" />
<bean id="connectionFactory" class="org.apache.activemq.spring.ActiveMQConnectionFactory">
<property name="brokerURL" value="${spring.activemq.broker-url}" />
<property name="trustAllPackages" value="true" />
</bean>
<int-jms:message-driven-channel-adapter id="jmsIn" destination-name="#{args[0]}" channel="requestsChannel" connection-factory="connectionFactory" max-messages-per-task="1"/>
<int:service-activator input-channel="requestsChannel" output-channel="replyChannel" ref="stepExecutionRequestHandler" />
<int:channel id="replyChannel" />
<int-jms:outbound-channel-adapter connection-factory="connectionFactory" destination-name="#{args[1]}" channel="replyChannel" />
Please advice if you have experience the issue.
Let me know if you need more information.
Note: I already search a lot at here and google but no luck for solution yet.
ActiveMQ uses a prefetch of 1000 by default see here.
In other words, the first (up to) 1000 partitions will go to the first consumer etc.
You can reduce the prefetch; 1 is probably fine for this application.
The performance of aggregator to process JMS messages along with persistence message store is very low. Like for a simple example given below the messages processed are between 25-28 msgs per second.
Is this the normal behavior? Or am I doing something wrong here ?
<!-- ###################### Inbound Message Adapter ####################### -->
<int-jms:message-driven-channel-adapter
id="xmlInboundMessageAdapter"
channel="msgUnmarshallingChannel"
connection-factory="jmsConnectionFactory"
destination="messsageQueue"
acknowledge="transacted"
concurrent-consumers="1"
max-concurrent-consumers="5"
auto-startup="true"/>
<!-- ###################### MSG UN-MARSHALLER ####################### -->
<int:channel id="msgUnmarshallingChannel" />
<int:chain input-channel="msgUnmarshallingChannel" output-channel="msgHeaderEnricherChannel">
<int-xml:unmarshalling-transformer unmarshaller="msgUnmarshaller" />
</int:chain>
<bean id="msgUnmarshaller" class="org.springframework.oxm.jaxb.Jaxb2Marshaller">
<property name="contextPath" value="com.msg.something" />
<property name="supportJaxbElementClass" value="true" />
</bean>
<int:channel id="msgHeaderEnricherChannel" />
<!-- ###################### SOME HEADER ENRICHMENT ####################### -->
<int:chain input-channel="msgHeaderEnricherChannel" output-channel="msgAggreggatorChannel">
<int:header-enricher>
<int:header name="CORRELATION_ID" expression="payload.notificationTypeId" />
</int:header-enricher>
</int:chain>
<int:channel id="msgAggreggatorChannel" />
<int:channel id="msgAggreggatorDiscardChannel" />
<!-- ###################### AGGREGATOR WITH PERSISTENCE MSG STORE ####################### -->
<int:aggregator
auto-startup="true"
send-timeout="-1"
message-store="messageStore"
input-channel="msgAggreggatorChannel"
output-channel="nullChannel"
discard-channel="msgAggreggatorDiscardChannel"
correlation-strategy="msgCorrelationStrategy"
release-strategy="msgReleaseStrategy"
expire-groups-upon-completion="true" />
<!-- ###################### MSG STORE ####################### -->
<bean id="messageStore" class="org.springframework.integration.jdbc.JdbcMessageStore">
<property name="dataSource" ref="dataSourceSPJDBC" />
<property name="lobHandler" ref="oracleLobHandler" />
</bean>
<bean id="oracleLobHandler" class="org.springframework.jdbc.support.lob.OracleLobHandler" />
<bean id="msgCorrelationStrategy" class="org.springframework.integration.aggregator.HeaderAttributeCorrelationStrategy">
<constructor-arg value="CORRELATION_ID" />
</bean>
<bean id="msgReleaseStrategy" class="org.springframework.integration.aggregator.MessageCountReleaseStrategy">
<constructor-arg value="10"/>
</bean>
It's a known problem for very large groups.
Which version of Spring Integration are you using? There have been a number of improvements made in this area, the latest being in 4.2.
The current version is 4.2.4; let us know if you're still seeing problems with that version.
We are using Spring Integration to consume messages from a queue and the requirement is to send the message to Error queue if there is any issue in the processing of the message consumed.
Flow works fine but one issue we see is when there is any exception thrown in processing of message, the message is redirected to the Error queue which we have configured but it is appended by the entire stack trace of the exception.
We are looking only for the original message to be delivered to the queue. Below is the configuration we have done,
<bean id="errorQ" class="com.ibm.mq.jms.MQQueue">
<constructor-arg value="${error.queue}" />
</bean>
<bean id="inQ" class="com.ibm.mq.jms.MQQueue">
<constructor-arg value="${inbound.queue}" />
</bean>
<int:channel id="error" />
<int:channel id="inbound" />
<int-jms:message-driven-channel-adapter
id="jmsIn" channel="inbound" container="messageListenerContainer"
acknowledge="transacted"></int-jms:message-driven-channel-adapter>
<int-jms:outbound-channel-adapter id="jmsError"
channel="error" connection-factory="mqConnectionFactory"
destination="errorQ" delivery-persistent="true"></int-jms:outbound-channel-adapter>
<int:service-activator id="service"
input-channel="inbound" ref="messageListener" method="someMethodInListerner">
<int:request-handler-advice-chain>
<ref bean="retryWithBackoffAdviceSession" />
</int:request-handler-advice-chain>
<bean id="retryWithBackoffAdviceSession"
class="org.springframework.integration.handler.advice.RequestHandlerRetryAdvice">
<property name="retryTemplate">
<bean class="org.springframework.retry.support.RetryTemplate">
<property name="backOffPolicy">
<bean class="org.springframework.retry.backoff.ExponentialBackOffPolicy">
<property name="initialInterval" value="${initialInterval}" />
<property name="multiplier" value="${multiplier}" />
<property name="maxInterval" value="${maxInterval}" />
</bean>
</property>
<property name="retryPolicy">
<bean class="org.springframework.retry.policy.SimpleRetryPolicy">
<property name="maxAttempts" value="${redelivery}" />
</bean>
</property>
</bean>
</property>
<property name="recoveryCallback">
<bean
class="org.springframework.integration.handler.advice.ErrorMessageSendingRecoverer">
<constructor-arg ref="error" />
</bean>
</property>
</bean>
The message sent to the error is an ErrorMessage with payload MessagingException.
The MessagingException has two properties cause and failedMessage.
So, if you want to send the original failed message to errorQ, you will need to add a transformer to the error flow...
<int:transformer ... expression="payload.failedMessage" />
EDIT
<int:chain input-channel="error">
<int:transformer expression="payload.failedMessage" />
<int-jms:outbound-channel-adapter ... />
</int:chain>
EDIT2
Generally, when using these techniques, it's useful to convey the reason for the failure. You can add the exception's message as a header...
<int:chain input-channel="error">
<int:header-enricher>
<int:header name="failureReason" expression="payload.cause.message" />
</int:header-enricher>
<int:transformer expression="payload.failedMessage" />
<int-jms:outbound-channel-adapter ... />
</int:chain>
A simple message-based RPC is very easy to create. The server side exports the service, the client side uses a proxy.
What is the best way, to make the same thing with multiple repliers?
I want to send a request from a client. Then the client waits while all (maybe with timeout) replies are received.
You could use an aggregator with appropriate correlation and release strategies (and a group timeout).
EDIT:
Here's a version using a JMS Topic...
<bean id="connectionFactory" class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory">
<bean class="org.apache.activemq.ActiveMQConnectionFactory">
<property name="brokerURL" value="vm://localhost"/>
</bean>
</property>
<property name="sessionCacheSize" value="10"/>
</bean>
<bean id="requestTopic" class="org.apache.activemq.command.ActiveMQTopic">
<constructor-arg value="topic.demo"/>
</bean>
<bean id="replyQueue" class="org.apache.activemq.command.ActiveMQQueue">
<constructor-arg value="queue.reply"/>
</bean>
<int-stream:stdin-channel-adapter id="stdin" channel="stdinToJmsoutChannel"/>
<int:channel id="stdinToJmsoutChannel"/>
<int:chain input-channel="stdinToJmsoutChannel">
<int:header-enricher>
<int:header name="jms_replyTo" ref="replyQueue" />
</int:header-enricher>
<int-jms:outbound-channel-adapter destination="requestTopic" />
</int:chain>
<int-jms:message-driven-channel-adapter channel="jmsReplyChannel"
destination="replyQueue"/>
<int:channel id="jmsReplyChannel" />
<int:chain input-channel="jmsReplyChannel">
<int:aggregator group-timeout="5000" expire-groups-upon-timeout="false"
send-partial-result-on-expiry="true"
discard-channel="logLateArrivers"
correlation-strategy-expression="headers['jms_correlationId']"
release-strategy-expression="size() == 2"/>
<int-stream:stdout-channel-adapter append-newline="true"/>
</int:chain>
<int:logging-channel-adapter id="logLateArrivers" />
<!-- Subscribers -->
<int-jms:inbound-gateway request-channel="upcase" request-destination="requestTopic" />
<int-jms:inbound-gateway request-channel="upcase" request-destination="requestTopic" />
<int:transformer input-channel="upcase" expression="payload.toUpperCase()" />
Type requests into the console:
Please type something and hit <enter>
foo
[FOO, FOO]
bar
[BAR, BAR]
baz
[BAZ, BAZ]
firstly thanks for attention
i define outbound-gateway ftp adapter to run ftp command on target server,my goal is run periodically ls,mv,get,... commands on server and run task on batch library job, my code is:
<bean id="ftpClientFactory1" class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="127.0.0.1"/>
<property name="port" value="21"/>
<property name="username" value="banks_reader"/>
<property name="password" value="123456"/>
<property name="clientMode" value="2"/>
<property name="fileType" value="2"/>
</bean>
<int:gateway id="gw" service-interface="ir.ali.util.ToFtpFlowGateway"
default-request-channel="inbound1"/>
<int:channel id="inbound1"/>
<int-ftp:outbound-gateway id="gateway1"
session-factory="ftpClientFactory1"
request-channel="inbound1"
reply-channel="outbound"
reply-timeout="777"
auto-create-local-directory="false"
auto-startup="true"
filename-pattern="*"
remote-file-separator="/"
command="ls"
command-options="-dirs -R"
expression="payload"
mput-regex=".*"
>
</int-ftp:outbound-gateway>
<int:channel id="outbound">
<int:interceptors>
<int:wire-tap channel="logger"/>
</int:interceptors>
</int:channel>
<int:channel id="outboundJobRequestChannel"/>
<int:logging-channel-adapter id="logger" log-full-message="true" />
<int:transformer input-channel="outbound" output-channel="outboundJobRequestChannel">
<bean class="ir.ali.configuration.FileMessageToJobRequest"/>
</int:transformer>
<int:splitter id="splitter" input-channel="outbound" output-channel="ftpChannel"/>
<int:channel id="ftpChannel">
<int:queue/>
</int:channel>
and in Application Class send message to inbound1 channel and start int-ftp:outbound-gateway the code is:
final ToFtpFlowGateway toFtpFlow = ctx.getBean(ToFtpFlowGateway.class);
try {
List<Boolean> rmResults = toFtpFlow.lsGetAndRmFiles("/");
} catch (Exception e) {
e.printStackTrace();
}
it's worked correctly , the problem is that i needed run periodically ls and mv ,... recursive command on the server,how to run toFtpFlow.lsGetAndRmFiles("/"); periodically to start int-ftp:outbound-gateway ?
<int:inbound-channel-adapter expression="/" channel="inbound1">
<int:poller fixed=delay="60000" />
</int:inbound-channel-adapter>
<int:channel id="inbound1"/>
<int-ftp:outbound-gateway id="gateway1"
session-factory="ftpClientFactory1"
request-channel="inbound1"
reply-channel="outbound"
reply-timeout="777"
auto-create-local-directory="false"
auto-startup="true"
filename-pattern="*"
remote-file-separator="/"
command="ls"
command-options="-dirs -R"
expression="payload"
mput-regex=".*"/>