We have camel route which reads from topic and then performs some transformation on message received it.
Given its durable topic only one message is processed at time, until route finishes its work.
To achieve concurrency, added threadpool so once message is received from topic to further work asynchronously, threads are spawning but its sequential one. Like once message is received thread is picked from pool and starts processing until that thread doesn't finish processing next message is not picked up.
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:camel="http://camel.apache.org/schema/spring"
xmlns:broker="http://activemq.apache.org/schema/core" xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://camel.apache.org/schema/spring
http://camel.apache.org/schema/spring/camel-spring.xsd
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core-5.5.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
<bean id="threadPool" class="java.util.concurrent.Executors" factory-method="newFixedThreadPool">
<constructor-arg index="0" value="10"/>
</bean>
<camel:camelContext id="camel-etl" trace="true"
xmlns="http://camel.apache.org/schema/spring">
<route id="topicRoute" errorHandlerRef="deadLetterErrorHandler" >
<from uri="{{inbound.topic}}"/>
<camel:threads executorServiceRef="threadPool">
<choice>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<when>
...
...
<multicast>
some loigc ...
</multicast>
<bean ref="persistData"/>
</when>
<otherwise>
...
<bean ref="deadLetterErrorHandler"/>
</otherwise>
</choice>
</camel:threads>
</route>
</camel:camelContext>
<!-- XSLT config -->
<bean id="saxonFactory" class="net.sf.saxon.TransformerFactoryImpl"/>
<!-- custom component beans -->
<bean id="persistData" class="com.data.PersistBean"/>
</beans>
You need to turn on asyncConsumer on the JMS endpoint. See the docs at: https://github.com/apache/camel/blob/master/components/camel-jms/src/main/docs/jms-component.adoc
Related
Below is my consumer and producer XML I can send message successfully on topic and can read from consumer but in console I am getting below error:
10823 [taskExecutor-30] ERROR
org.springframework.integration.handler.LoggingHandler -
org.springframework.messaging.MessagingException: Consuming from Kafka
failed; nested exception is java.util.concurrent.ExecutionException:
java.lang.IllegalStateException: Iterator is in failed state
Consumer XML
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:int="http://www.springframework.org/schema/integration"
xmlns:stream="http://www.springframework.org/schema/integration/stream"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/integration/stream http://www.springframework.org/schema/integration/stream/spring-integration-stream.xsd
http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration-4.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
<int:channel id="inputFromKafka">
<int:queue />
</int:channel>
<int:service-activator input-channel="inputFromKafka"
ref="messageProcessor" method="reader" >
<int:poller fixed-rate="50" task-executor="taskExecutor" >
</int:poller>
</int:service-activator>
<bean id="consumerProperties"
class="org.springframework.beans.factory.config.PropertiesFactoryBean">
<property name="properties">
<props>
<prop key="auto.offset.reset">largest</prop>
<prop key="socket.receive.buffer.bytes">10485760</prop>
<prop key="fetch.message.max.bytes">5242880</prop>
<prop key="auto.commit.interval.ms">100000</prop>
</props>
</property>
</bean>
<int-kafka:zookeeper-connect id="zookeeperConnect"
zk-connect="localhost:2181" zk-connection-timeout="6000"
zk-session-timeout="6000" zk-sync-time="2000" />
<bean id="kafkaThreadListener" class="com.maistylz.adminui.integration.ConsumerStarter"
init-method="initIt" />
<int-kafka:inbound-channel-adapter
kafka-consumer-context-ref="consumerContext" auto-startup="true"
channel="inputFromKafka" id="kafka-inbound-channel-adapter" >
<int:poller fixed-delay="1000" time-unit="MILLISECONDS"
receive-timeout="1000" />
</int-kafka:inbound-channel-adapter>
<task:executor id="taskExecutor" pool-size="50" keep-alive="120" queue-capacity="500"/>
<int-kafka:consumer-context id="consumerContext"
consumer-timeout="400000" zookeeper-connect="zookeeperConnect"
consumer-properties="consumerProperties" >
<int-kafka:consumer-configurations>
<int-kafka:consumer-configuration
group-id="default" max-messages="500" executor="taskExecutor">
<int-kafka:topic id="event-stream" streams="1" />
</int-kafka:consumer-configuration>
</int-kafka:consumer-configurations>
</int-kafka:consumer-context>
</beans>
Producer XML
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
xmlns:task="http://www.springframework.org/schema/task"
xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
<int:channel id="inputToKafka">
<int:queue/>
</int:channel>
<bean id="producerProperties"
class="org.springframework.beans.factory.config.PropertiesFactoryBean">
<property name="properties">
<props>
<prop key="topic.metadata.refresh.interval.ms">360000000</prop>
<prop key="message.send.max.retries">5</prop>
<prop key="send.buffer.bytes">5242880</prop>
</props>
</property>
</bean>
<bean id="encoder"
class="org.springframework.integration.kafka.serializer.common.StringEncoder" />
<int-kafka:outbound-channel-adapter
id="kafkaOutboundChannelAdapter" auto-startup="true"
kafka-producer-context-ref="kafkaProducerContext"
channel="inputToKafka" >
<int:poller fixed-delay="1000" receive-timeout="0" task-executor="taskExecutor"/>
</int-kafka:outbound-channel-adapter>
<task:executor id="taskExecutor" pool-size="5" keep-alive="1200" queue-capacity="10000"/>
<int-kafka:producer-context id="kafkaProducerContext">
<int-kafka:producer-configurations>
<int-kafka:producer-configuration broker-list="localhost:9092"
topic="event-stream"
key-class-type="java.lang.String"
value-class-type="java.lang.String"
value-encoder="encoder"
key-encoder="encoder"
/>
</int-kafka:producer-configurations>
</int-kafka:producer-context>
</beans>
I have generated the java classes from a wadl file with cxf. There are 3 resources definded an so 3 service classes with #PATH annotation are created. Now I want to publish them to the same url but I'm not sure how to achieve this.
Here are the snippets of the classes and wadl. The last part shows the beans.xml - At this point this is the only way I know how to publish the endpoints. Is there anothher way and how can I publish these 3 classes to the base url "/" and then they should match to the paths related to the annotations. Maybe a wrapper class for all but I'm not sure?
classes
#Path("status")
public class Status {
...
#Path("status/{id}")
public class StatusId {
...
#Path("counters")
public class Counters{
...
wadl
<resources base="http:localhost:8080/rest">
<resource path="status/{id}" id="status">
<method name="GET" id="getStatusById">
...
<resource path="status" id="status">
<method name="GET" id="getStatusByQueryParam">
...
</resource>
<resource path="counters" id="counters">
<method name="PUT" id="putCounters">
...
beans.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jaxrs="http://cxf.apache.org/jaxrs"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://cxf.apache.org/jaxrs
http://cxf.apache.org/schemas/jaxrs.xsd">
<!-- do not use import statements if CXFServlet init parameters link to this beans.xml -->
<import resource="classpath:META-INF/cxf/cxf.xml" />
<import resource="classpath:META-INF/cxf/cxf-servlet.xml" />
<jaxrs:server id="statusService" address="/">
<jaxrs:serviceBeans>
<ref bean="status" />
</jaxrs:serviceBeans>
</jaxrs:server>
<!-- causes error
<jaxrs:server id="statusServiceId" address="/">
<jaxrs:serviceBeans>
<ref bean="statusId" />
</jaxrs:serviceBeans>
</jaxrs:server>
<jaxrs:server id="counterServiceId" address="/">
<jaxrs:serviceBeans>
<ref bean="counters" />
</jaxrs:serviceBeans>
</jaxrs:server>
-->
<bean id="status" class="package.Status"/>
<bean id="statusId" class="package.StatusId"/>
<bean id="counters" class="package.Counters"/>
</beans>
for geting multiple endpoints your
beans.xml should be looks like bellow
<jaxrs:server id="statusService" address="/">
<jaxrs:serviceBeans>
<ref bean="status" />
<ref bean="statusId" />
<ref bean="counters" />
</jaxrs:serviceBeans>
</jaxrs:server>
I am creating a text message and put it in the activemq queue and i display it in the log. Now i need to pass this message to the cxf rs client to use it in the parameter. I am using blueprint to define the camel routes and cxf client.
<?xml version="1.0" encoding="UTF-8"?>
<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
default-activation="eager" xmlns:xsi="http://www.w3.org/2001/XMLSchema- instance"
xmlns:jaxws="http://cxf.apache.org/blueprint/jaxws" xmlns:cxf="http://camel.apache.org/schema/blueprint/cxf"
xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:jaxrs="http://cxf.apache.org/blueprint/jaxrs"
xmlns:http="http://cxf.apache.org/transports/http/configuration"
xsi:schemaLocation="
http://www.osgi.org/xmlns/blueprint/v1.0.0
http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd
http://camel.apache.org/schema/blueprint/cxf
http://camel.apache.org/schema/blueprint/cxf/camel-cxf.xsd
http://cxf.apache.org/blueprint/jaxrs
http://cxf.apache.org/schemas/blueprint/jaxrs.xsd
http://cxf.apache.org/configuration/security
http://cxf.apache.org/schemas/configuration/security.xsd
http://cxf.apache.org/transports/http/configuration
http://cxf.apache.org/schemas/configuration/http-conf.xsd">
<!-- Beans -->
<bean id="myTransform" class="cxfcamel.MyTransform"/>
<bean id="serviceBean" class="cxfcamel.GreetingService" />
<bean id="rsprocessor" class="cxfcamel.RSProcessor"/>
<!-- Web Services -->
<jaxrs:server id="customerService" address="http://localhost:7171 /customers">
<jaxrs:serviceBeans>
<ref component-id="serviceBean" />
</jaxrs:serviceBeans>
</jaxrs:server>
<cxf:rsClient id="rsClient"
address="http://localhost:7171/customers/entry-point/register/nosJ"
serviceClass="cxfcamel.GreetingService">
</cxf:rsClient>
<!-- Camel Routes -->
<camelContext id="camel"
xmlns="http://camel.apache.org/schema/blueprint">
<route>
<from uri="timer://projectTimer?repeatCount=1" />
<bean ref="myTransform" method="transform" />
<to uri="activemq:queue:LOG.ME" />
</route>
<route>
<from uri="activemq:queue:LOG.ME" />
<to uri="log:ExampleActiveMQRouterBlueprint" />
</route>
<route>
<from uri="activemq:queue:LOG.ME" />
<setHeader headerName="Content-Type">
<constant>application/json</constant>
</setHeader>
<setHeader headerName="CamelHttpMethod">
<constant>PUT</constant>
</setHeader>
<to uri="cxfrs:bean:rsClient" />
</route>
</camelContext>
Can any one help me please?
Thanks
Both routes listen to activemq:queue:LOG.ME . A queue in ActiveMQ will consume the message and any other queue will not receive the message. You need to do 1 of 2 things:
Turn your queue into a topic so that both routes will receive the message. Topic vs Queue
Arrange your routes so only one route is listening to activemq:queue:LOG.ME.
There are two ways to accomplish this:
Turn your cxfrs:bean:rsClient invocation into a cxfrs:http://localhost:7171/customers/entry-point/register/nosJ and append the parameters at the end.
The documentation on this isn't very clear, but you might be able to use setHeader:
<setHeader headerName="CamelCxfRsQueryMap">
expression which returns a map
</setHeader>
This expression can be <bean>, a <groovy> embedded expression, etc.
I am trying to set a variable inside of my camel code so I can call this variable when I log my route execution. This variable needs to be set from an xpath statement.
Below is the code that is not working and I suspect that I need to set a variable that is equal to my xpath statement that is found in the log message but I don't know how to do that.
<?xml version="1.0" encoding="UTF-8"?>
<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:camel="http://camel.apache.org/schema/blueprint"
xsi:schemaLocation="
http://www.osgi.org/xmlns/blueprint/v1.0.0 http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd
http://camel.apache.org/schema/blueprint http://camel.apache.org/schema/blueprint/camel-blueprint.xsd">
<bean id="activemq"
class="org.apache.activemq.camel.component.ActiveMQComponent">
<property name="brokerURL" value="tcp://localhost:61616"/>
<property name="userName" value="user"/>
<property name="password" value="password"/>
</bean>
<camelContext id="blueprintContext" trace="false" xmlns="http://camel.apache.org/schema/blueprint">
<route id="Test_Message_Content_Route">
<from uri="activemq:queue:FirstQueue?username=user&password=password&concurrentConsumers=1&maxConcurrentConsumers=5"/>
<choice>
<when>
<xpath>//destination[text()='TEST']</xpath>
<log message="Test route invoked"/>
<split>
<xpath resultType="java.lang.String">//message_payload/text()</xpath>
<log message="Routed $xpath{//id/text()} to TEST QUEUE"/>
<to uri="activemq:queue:TestQueue?username=user&password=password"/>
</split>
</when>
<when>
<xpath>//destination[text()='DEV']</xpath>
<log message="Dev route invoked"/>
<split>
<xpath resultType="java.lang.String">//message_payload/text()</xpath>
<log message="Routed $xpath{//id/text()} to DEV QUEUE"/>
<to uri="activemq:queue:DevQueue?username=user&password=password"/>
</split>
</when>
<otherwise>
<log message="Sending message to DL Queue"/>
<to uri="activemq:queue:DLQueue?username=user&password=password"/>
</otherwise>
</choice>
</route>
</camelContext>
</blueprint>
I now have this working by using setHeader in my camel context like the following:
<setHeader headerName="id"><xpath>//id/text()</xpath></setHeader>
<log message="Routed ${header.id} to Test queue"/>
I am trying to implement routing solution in which i need to read messages from one queue of rabbit mq and put message on different rabbit mq/different queue.
I was able to make read and write separately from queues but it is not working together.
Here is camel configuration file :
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rabbit="http://www.springframework.org/schema/rabbit"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd
http://www.springframework.org/schema/rabbit http://www.springframework.org/schema/rabbit/spring-rabbit-1.0.xsd">
<context:annotation-config/>
<context:component-scan base-package="amqp.spring.camel.component"/>
<camelContext xmlns="http://camel.apache.org/schema/spring">
<jmxAgent id="agent" createConnector="false" disabled="true" />
<template id="wfcTemplate" />
<template id="routerTemplate" />
<route>
<from uri="spring-amqp:exchange1:queue1:ABCD?type=topic&autodelete=false&durable=true" />
<log message="Message available on a RabbitMQ Queue : ${body}" />
<to uri="spring-amqp:exchange2:queue2:EFGH?type=topic&autodelete=false&durable=true" />
</route>
</camelContext>
<rabbit:connection-factory id="producerConnectionFactory" connection-factory="producerConnectionFactory" />
<rabbit:template id="routerTemplate" connection-factory="producerConnectionFactory" message-converter="messageConverter" exchange="exchange2" />
<rabbit:admin connection-factory="producerConnectionFactory" />
<bean id="producerConnectionFactory" class="org.springframework.amqp.rabbit.connection.CachingConnectionFactory">
<property name="host" value="10.0.10.100"/>
<property name="port" value="5672"/>
<property name="username" value="shailesh"/>
<property name="password" value="shailesh"/>
<property name="virtualHost" value="vh1"/>
</bean>
<rabbit:connection-factory id="consumerConnectionFactory" connection-factory="consumerConnectionFactory"/>
<rabbit:template id="wfcTemplate" connection-factory="consumerConnectionFactory" message-converter="messageConverter" exchange="exchange1" />
<rabbit:admin connection-factory="consumerConnectionFactory"/>
<bean id="consumerConnectionFactory" class="org.springframework.amqp.rabbit.connection.CachingConnectionFactory">
<property name="host" value="10.0.10.101"/>
<property name="port" value="5672"/>
<property name="username" value="shailesh"/>
<property name="password" value="shailesh"/>
<property name="virtualHost" value="vh2"/>
</bean>
<!-- converters -->
<bean id="jsonMessageConverter" class="amqp.spring.converter.XStreamConverter"/>
<bean id="textMessageConverter" class="amqp.spring.converter.StringConverter"/>
<bean id="messageConverter" class="amqp.spring.converter.ContentTypeConverterFactory">
<property name="converters">
<map>
<entry key="application/json" value-ref="jsonMessageConverter"/>
<entry key="application/xml" value-ref="textMessageConverter"/>
</map>
</property>
<property name="fallbackConverter" ref="textMessageConverter"/>
</bean>
</beans>
There error I am getting is as follows :
[pache.camel.spring.Main.main()] SpringCamelContext INFO Total 1 routes, of which 1 is started.
[pache.camel.spring.Main.main()] SpringCamelContext INFO Apache Camel 2.10.3 (CamelContext: camel-1) started in 0.589 seconds
[PConsumer.SpringAMQPExecutor-1] route1 INFO Message available on a RabbitMQ Queue : Hello, world! Fri Feb 08 14:14:33 CST 2013
[l-1) thread #0 - amqp-producer] SpringAMQPProducer ERROR Could not deliver message via AMQP
java.lang.NullPointerException
at amqp.spring.camel.component.SpringAMQPProducer$AMQPProducerTask.run(SpringAMQPProducer.java:150)[camel-spring-amqp-1.4.jar:]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)[:1.7.0_11]
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)[:1.7.0_11]
at java.util.concurrent.FutureTask.run(FutureTask.java:166)[:1.7.0_11]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)[:1.7.0_11]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)[:1.7.0_11]
at java.lang.Thread.run(Thread.java:722)[:1.7.0_11]