MultiResourceItemReader not working as expected - spring-boot

Question

I have been using FlatFileItemReader and processing files one by one. But I am trying to use MultiResourceItemReader and supplying all the files at once, there are 3 CSV files after filtering and at most 50.
While running the Job even if all the files are supplied, if I verify the results only 1 file is processed. Data is read from the CSV file and saved to database and after verifying the results only the data of 1 file is saved to database. I couldn't find what am I doing wrong. My code for
MultiResourceItemReader is below :
#Bean(name = "multiItemReader")
#StepScope
public MultiResourceItemReader<CDSBrokerBOIDMappingEntity> multiResourceItemReader(#Value("#{jobParameters[filenameStartPattern]}") String filenameStartPattern
, #Value("#{jobParameters[filenameEndPattern]}") String filenameEndPattern, #Value("#{jobParameters[localDirectory]}") String localDirectory) throws Exception {
String[] localDirectories = localDirectory.split(",");
List<Resource> inputResources = Collections.synchronizedList(new ArrayList<>());
for (String localDirectory1 : localDirectories){
try (Stream<Path> walk = Files.walk(Paths.get(localDirectory1), 1)) {
walk.filter(Files::isRegularFile) // is a file
.filter(p -> p.getFileName().toString().startsWith(filenameStartPattern) && p.getFileName().toString().endsWith(filenameEndPattern))
.findAny().ifPresentOrElse(f -> {
log.info("CSV FILE => " + f.getFileName().toString());
inputResources.add(new FileSystemResource(f));
},
() -> {
log.info("No file found");
});
} catch (IOException e) {
e.printStackTrace();
}
}
log.info("No. of files => "+inputResources.size());
MultiResourceItemReader<CDSBrokerBOIDMappingEntity> resourceItemReader = new MultiResourceItemReader<CDSBrokerBOIDMappingEntity>();
resourceItemReader.setResources(inputResources.toArray(Resource[]::new));
resourceItemReader.setDelegate(importReader());
resourceItemReader.setStrict(true);
return resourceItemReader;
}
And FlatFileItemReader code is :
#Bean
public FlatFileItemReader<CDSBrokerBOIDMappingEntity> importReader() throws Exception {
FlatFileItemReader<CDSBrokerBOIDMappingEntity> reader = new FlatFileItemReader<>();
reader.setLinesToSkip(1);
reader.setLineMapper(new DefaultLineMapper<CDSBrokerBOIDMappingEntity>() {{
setLineTokenizer(new DelimitedLineTokenizer() {{
setNames(new String[]{"BOID", "CLIENT_MEMBER_CODE", "BROKER_ID", "LAST_MODIFIED_DATE", "ISVALID"});
}});
setFieldSetMapper(new BeanWrapperFieldSetMapper<CDSBrokerBOIDMappingEntity>() {{
setTargetType(CDSBrokerBOIDMappingEntity.class);
}});
}});
reader.setStrict(true);
reader.afterPropertiesSet();
return reader;
}
Writer is :
#Bean
public ItemWriter<CDSBrokerBOIDMappingEntity> writer() {
// log.info("Writer current thread. {}", Thread.currentThread().getName());
RepositoryItemWriter<CDSBrokerBOIDMappingEntity> writer = new RepositoryItemWriter<CDSBrokerBOIDMappingEntity>();
writer.setRepository(cdsBrokerBOIDMappingRepository);
// writer.setMethodName("save");
try {
writer.afterPropertiesSet();
} catch (Exception e) {
e.printStackTrace();
}
return writer;
}
And Step and Job :
#Bean
public Job importUserJob(MultiResourceItemReader<CDSBrokerBOIDMappingEntity> importReader, JobCompletionNotificationListener listener) {
return jobBuilderFactory
.get("importUserJob")
.incrementer(new RunIdIncrementer())
.listener(listener)
.flow(step1(importReader))
.end()
.build();
}
#Bean
public Step step1(#Qualifier("multiItemReader") MultiResourceItemReader<CDSBrokerBOIDMappingEntity> importReader) {
return stepBuilderFactory.get("step1").<CDSBrokerBOIDMappingEntity, CDSBrokerBOIDMappingEntity>chunk(200)
.reader(importReader)
.processor(processor())
.writer(writer())
.listener(stepListener())
.taskExecutor(taskExecutor())
.build();
}
I tried multiple times but only 1 file is read.
Is there anything wrong in the code ? Or my approach is wrong ?

Answer 1

The MultiResourceItemReader is working as expected with Spring Batch v4.3.3, here is a quick example:
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
import org.springframework.batch.item.file.mapping.PassThroughLineMapper;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
#Configuration
#EnableBatchProcessing
public class SO68125366 {
#Bean
public MultiResourceItemReader<String> itemReader() {
FlatFileItemReader<String> itemReader = new FlatFileItemReaderBuilder<String>()
.name("itemReader")
.lineMapper(new PassThroughLineMapper())
.build();
MultiResourceItemReader<String> multiResourceItemReader = new MultiResourceItemReader<>();
multiResourceItemReader.setDelegate(itemReader);
Resource resource1 = new FileSystemResource("file1.txt");
Resource resource2 = new FileSystemResource("file2.txt");
multiResourceItemReader.setResources(new Resource[] {resource1, resource2});
return multiResourceItemReader;
}
#Bean
public ItemWriter<String> itemWriter() {
return items -> items.forEach(System.out::println);
}
#Bean
public Job job(JobBuilderFactory jobs, StepBuilderFactory steps) {
return jobs.get("job")
.start(steps.get("step")
.<String, String>chunk(5)
.reader(itemReader())
.writer(itemWriter())
.build())
.build();
}
public static void main(String[] args) throws Exception {
ApplicationContext context = new AnnotationConfigApplicationContext(SO68125366.class);
JobLauncher jobLauncher = context.getBean(JobLauncher.class);
Job job = context.getBean(Job.class);
jobLauncher.run(job, new JobParameters());
}
}
With two files file1.txt and file2.txt containing respectively hello and world, the sample prints:
hello
world
which means the MultiResourceItemReader read both resources and not only one as you mention. The complete sample can be found in this repo.