How to stop file transferring in spring-batch - spring

Question

I have created a spring-batch job for reading files from local directory and upload it to remote directory through ftp using Camel-spring-batch. I am doing the same using chunk.
My spring batch job configuration looks like :
<bean id="consumerTemplate" class="org.apache.camel.impl.DefaultConsumerTemplate" init-method="start" destroy-method="stop">
<constructor-arg ref="camelContext"/>
</bean>
<bean id="producerTemplate" class="org.apache.camel.impl.DefaultProducerTemplate" scope="step" init-method="start" destroy-method="stop">
<constructor-arg ref="camelContext"/>
</bean>
<bean id="localFileReader" class="com.camel.springbatch.reader.LocalFileReader" scope="step" destroy-method="stop">
<constructor-arg value="file:#{jobParameters['dirPath']}"/>
<constructor-arg ref="consumerTemplate"/>
</bean>
<bean id="ftpFileWriter" class="com.camel.springbatch.writer.FtpFileWriter" scope="step">
<constructor-arg ref="producerTemplate"/>
<constructor-arg value="ftp://#{jobParameters['host']}?username=#{jobParameters['user']}&password=#{jobParameters['password']}"/>
</bean>
Job configuration :
<batch:job id="ftpReadWrite">
<batch:step id="readFromLocalWriteToFtp" next="readFromFtpWriteToLocal">
<batch:tasklet>
<batch:chunk reader="localFileReader" writer="ftpFileWriter" commit-interval="5" />
</batch:tasklet>
</batch:step>
And my "Localfilereader" and "ftpFileWriter" looks like :
import org.apache.camel.ConsumerTemplate;
import org.apache.camel.component.spring.batch.support.CamelItemReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class LocalFileReader extends CamelItemReader {
private Logger log= LoggerFactory.getLogger(this.getClass());
ConsumerTemplate consumerTemplate;
String endpointUri;
public LocalFileReader(ConsumerTemplate consumerTemplate, String endpointUri) {
super(consumerTemplate, endpointUri);
this.consumerTemplate=consumerTemplate;
this.endpointUri=endpointUri;
}
#Override
public Object read() throws Exception {
Object item = consumerTemplate.receiveBody(endpointUri);
return item;
}
}
"Ftp File Writer"
import org.apache.camel.ProducerTemplate;
import org.apache.camel.component.spring.batch.support.CamelItemWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
public class FtpFileWriter extends CamelItemWriter {
private Logger log= LoggerFactory.getLogger(this.getClass());
ProducerTemplate producerTemplate;
String endpointUri;
public FtpFileWriter(ProducerTemplate producerTemplate, String endpointUri) {
super(producerTemplate, endpointUri);
this.producerTemplate=producerTemplate;
this.endpointUri=endpointUri;
}
#Override
public void write(List items) throws Exception {
System.out.println("************************Writing item to ftp "+items);
for (Object item : items) {
System.out.println("writing item [{}]..."+item);
producerTemplate.sendBody(endpointUri, item);
log.debug("wrote item");
}
}
}
It works fine if I have only 5 file in my local directory. It read the all 5 file from my local directory and it send to the writer and writer send it to the ftp server as my commit-interval=5. If I have 6 file in in local directory then it send first chunk of 5 file to writer and again it start reading the remaining file and this time there is only one file remaining. It read 1 file and start waiting for 4 files and never send to writer. I tried it with commit-interval=1 now it send all 6 files to server and again start waiting for next file. Here I need to stop the process once all file have been processed.
Please help me to resolved this issue...

Answer 1

From ConsumerTemplate's javadoc receiveBody waits until there is a response; you need to work with timeout (check TimeoutPolicy in spring-batch) or a different way to mark reader as 'exhausted' (return null from reader) to stop reader from reading

Answer 2

You could use receiveBodyNoWait instead of receiveBody.
Then you have to check if there are files remaining inside the consumer endpoint. I code this for a tasklet that consumes big-xml file into smaller pieces.
The tasklet :
public class MyCamelTasklet extends ServiceSupport implements Tasklet, InitializingBean{
private static final Logger LOG = LoggerFactory.getLogger(MyCamelTasklet.class);
private final CamelContext camelContext;
private final ConsumerTemplate consumerTemplate;
private final File workingDir;
private final Route xmlSplitRoute;
public MyCamelTasklet(ConsumerTemplate consumerTemplate) {
super();
this.consumerTemplate = consumerTemplate;
this.camelContext = consumerTemplate.getCamelContext();
this.xmlSplitRoute = this.camelContext.getRoutes().get(0);
this.workingDir = new File(xmlSplitRoute.getRouteContext().getFrom().getUri().replace("file:", ""));
}
#Override
public RepeatStatus execute(StepContribution arg0, ChunkContext arg1)
throws Exception {
LOG.debug("reading new item...");
Endpoint endpointXmlSplitRoute = xmlSplitRoute.getEndpoint();
while(getNbFilesToConsume(this.workingDir) > 0) {
consumerTemplate.receiveBodyNoWait(endpointXmlSplitRoute);
}
return RepeatStatus.FINISHED;
}
private int getNbFilesToConsume(File workingDir){
return FileUtils.listFiles(workingDir, new String[]{"xml"}, false).size();
}
#Override
protected void doStart() throws Exception {
ServiceHelper.startService(consumerTemplate);
}
#Override
protected void doStop() throws Exception {
ServiceHelper.stopService(consumerTemplate);
}
#Override
public void afterPropertiesSet() throws Exception {
ObjectHelper.notNull(camelContext, "CamelContext", this);
camelContext.addService(this);
}
}
The unit test for the preceding tasklet:
public class SplitTaskletTest {
#Test public void execute() throws Exception {
CamelContext camelContext = new DefaultCamelContext();
camelContext.addRoutes(new RouteBuilder() {
public void configure() {
Namespaces ns = new Namespaces("nsl", "http://www.toto.fr/orders");
from("file:data/inbox").id("inbox-road").
split().
xtokenize("//nsl:order", 'w', ns, 1000).
streaming().
to("file:data/outbox?fileName=${file:name.noext}-${exchangeId}.${file:ext}");
}
});
camelContext.start();
ConsumerTemplate consumer =new DefaultConsumerTemplate(camelContext);
consumer.start();
MyCamelTasklet tasklet = new MyCamelTasklet(consumer);
long debutTraitement = System.currentTimeMillis();
tasklet.execute(null, null);
long finTraitement = System.currentTimeMillis();
long total = finTraitement-debutTraitement;
File outputDir = new File("data/outbox");
outputDir.mkdir();
int nbGeneratesFiles = FileUtils.listFiles(outputDir, new String[]{"xml"}, false).size();
System.out.println("Traitement total en secondes : "+total/1000);
Assert.assertTrue(nbGeneratesFiles>0);
}
}