Springboot Batch get CSV from rest API - spring-boot

Question

I have a scenario for Springboot batch application where I need to read the CSV-11 and write to another CSV-2, the problem arises when I need to get the CSV-1 from a REST endpoint, for e.g. when I initiate the job it should fetch a CSV-1 from an endpoint and then continue the batch processing to write to CSV-2.
But it seems like to process a CSV-1 we need to feed the 'Resource' in advance while application startup [I am fairly new to batch so not sure if it's 100% correct].
Can anyone please guide me the right approach to resolve this?
EDIT : Adding code, I was able to resolve it but need advice if it's the right way to do things (please ignore the hard-coded data).
#Configuration
#EnableBatchProcessing
public class BatchConfig {
#Autowired
private JobBuilderFactory jobbuilder;
#Autowired
private StepBuilderFactory stepbuilder;
#Autowired
private CSVProcessor processor;
#Autowired
private CustomSkipPolicy customSkipPolicy;
#Autowired
private CSVResponse response;
#Bean(name="csv")
public Job job_csv() throws Exception {
Step step = stepbuilder
.get("csv-step")
.<Person, Person>chunk(5)
.reader(new CSVReader(response.getResource()))
.processor(processor)
.writer(new CSVWriter().write(response.getFilename()))
.faultTolerant()
.skipPolicy(customSkipPolicy)
.listener(new StepListener())
.build();
return jobbuilder
.get("csv-job")
.incrementer(new RunIdIncrementer())
.listener(new JobListener())
.flow(step)
.end()
.build();
}
}
#Slf4j
public class CSVReader extends FlatFileItemReader<Person> {
public CSVReader(Resource resource) throws Exception {
super();
setResource(resource);
setStrict(false);
setLinesToSkip(1);
doOpen();
DelimitedLineTokenizer dlt = new DelimitedLineTokenizer();
dlt.setNames(new String[] {"id","first_name","last_name","email","gender","ip_address","dob"});
dlt.setDelimiter(",");
dlt.setStrict(false);
BeanWrapperFieldSetMapper<Person> fsp = new BeanWrapperFieldSetMapper<>();
fsp.setTargetType(Person.class);
DefaultLineMapper<Person> dlp = new DefaultLineMapper<>();
dlp.setLineTokenizer(dlt);
dlp.setFieldSetMapper(fsp);
setLineMapper(dlp);
}
}
public class CSVWriter {
private static final String DATA_PROCESSED = "C:/data/processed";
public FlatFileItemWriter<Person> write(String filename) throws Exception {
FlatFileItemWriter<Person> writer = new FlatFileItemWriter<>();
writer.setResource(resource(filename));
writer.setHeaderCallback(new Header());
BeanWrapperFieldExtractor<Person> fe = new BeanWrapperFieldExtractor<>();
fe.setNames(new String[] { "id", "first_name", "last_name", "age" });
DelimitedLineAggregator<Person> dla = new DelimitedLineAggregator<>();
dla.setDelimiter(",");
dla.setFieldExtractor(fe);
writer.setLineAggregator(dla);
return writer;
}
private Resource resource(String filename) throws IOException {
String processed_file = filename.replace(".csv", "").concat("_PROCESSED").concat(".csv");
if (Files.notExists(Paths.get(DATA_PROCESSED), new LinkOption[] { LinkOption.NOFOLLOW_LINKS })) {
Files.createDirectory(Paths.get(DATA_PROCESSED));
}
if (Files.notExists(Paths.get(DATA_PROCESSED + "/" + processed_file),
new LinkOption[] { LinkOption.NOFOLLOW_LINKS })) {
Files.createFile(Paths.get(DATA_PROCESSED + "/" + processed_file));
}
return new FileSystemResource(DATA_PROCESSED + "/" + processed_file);
}
}
#Component
#Slf4j
public class CSVResponse {
private static final String DATA_RECEIVE = "C:/data/receive";
private String filename;
public Resource getResource() throws IOException {
ResponseEntity<Resource> resp = new RestTemplate().getForEntity("http://localhost:8081/csv", Resource.class);
log.info("File Received : "+resp.getBody().getFilename());
setFilename(resp.getBody().getFilename());
Path path = Paths.get(DATA_RECEIVE);
if(!Files.exists(path, new LinkOption[]{ LinkOption.NOFOLLOW_LINKS})) {
Files.createDirectory(path);
}
Files.copy(resp.getBody().getInputStream(), Paths.get(DATA_RECEIVE + "/" +resp.getBody().getFilename()), StandardCopyOption.REPLACE_EXISTING);
return resp.getBody();
}
public String getFilename() {
return filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
}

Answer 1

You can implement item reader with StepExecutionListener interface, and you have to implement the fetching CSV-1 file via RestTemplate in beforeStep() method of the item reader.
You can refer how to use RestTemplate from https://www.baeldung.com/rest-template