Files are sending to wrong sftp location while using Spring SFTP outbound gateway - spring

We are using Spring SFTP (outbound) using Gateway to transfer files to multiple destinations. But often, few files are sent to the wrong destination. Can't find any clue as we don't get any error in our log except file count error after sending files.
Here is our configuration:
#Configuration
public class BankWiseSFTPConfig {
private final ExpressionParser EXPRESSION_PARSER;
private final BankConfigService bankConfigService;
public BankWiseSFTPConfig(BankConfigService bankConfigService) {
this.EXPRESSION_PARSER = new SpelExpressionParser();
this.bankConfigService = bankConfigService;
}
#Bean
public DelegatingSessionFactory<LsEntry> sessionFactory() {
List<BankConfigEntity> bankList = bankConfigService.getAll();
Map<Object, SessionFactory<LsEntry>> factories = new LinkedHashMap<>();
for (BankConfigEntity bank : bankList) {
DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory();
factory.setHost(bank.getSftpHost());
factory.setUser(bank.getSftpUser());
factory.setPort(bank.getSftpPort());
factory.setPassword(bank.getSftpPass());
factory.setAllowUnknownKeys(true);
factories.put(bank.getBankName(), factory);
}
bankList.clear();
return new DelegatingSessionFactory<LsEntry>(factories, factories.values().iterator().next());
}
#ServiceActivator(inputChannel = "toSftp")
#Bean
public SftpMessageHandler handler() {
SftpMessageHandler handler = new SftpMessageHandler(new SftpRemoteFileTemplate(sessionFactory()));
handler.setRemoteDirectoryExpression(EXPRESSION_PARSER.parseExpression("headers['path']"));
return handler;
}
#MessagingGateway
public interface SFTPOutboundGateway {
#Gateway(requestChannel = "toSftp")
void push(File file, #Header("path") String path);
#Gateway(requestChannel = "sftpChannel")
List<String> executeCommand(String path);
}
#Bean
#ServiceActivator(inputChannel = "sftpChannel")
public MessageHandler messageHandlerLs() {
SftpOutboundGateway sftpOutboundGateway = new SftpOutboundGateway(sessionFactory(), "ls", "payload");
sftpOutboundGateway.setOptions("-1 -R");
return sftpOutboundGateway;
}
}
Here are our push and file count methods:
private void pushReport(String bankName,
String destinationPath,
String sourcePath,
String refundType,
List<BankReportEntity> failedBankReportEntities,
List<BankReportEntity> pushedFiles,
BankReportEntity bankReportEntity) {
String sftpStatus = SlotBankStatus.BANK_SFTP_INITIATED.name();
String errorReason = StringUtils.EMPTY;
String fileName = bankReportEntity.getFileName();
String filePath = sourcePath + fileName;
File file = new File(filePath);
bankReportEntity.setSftpStatus(sftpStatus);
log.debug("{} :: SFTP Push Initiated for {} and File {}", refundType, bankName, fileName);
try {
log.info("{} :: SFTP Push trying for {} and {}", refundType, bankName, file);
gateway.push(file, destinationPath);
sftpStatus = SlotBankStatus.BANK_SFTP_COMPLETED.name();
pushedFiles.add(bankReportEntity);
log.info("{} :: SFTP Push success for {} and {}", refundType, bankName, file);
} catch (Exception e) {
emailService.sendSFTPExceptionEmail(
"File push error for file : " + fileName +
" and FileTransferType " + bankReportEntity.getFileTransferType() +
". Error : " + e.getLocalizedMessage(),
bankName);
sftpStatus = SlotBankStatus.BANK_SFTP_PENDING.name();
errorReason = ErrorCode.SFTP_PUSH_FAILED.name();
failedBankReportEntities.add(bankReportEntity);
log.error("{} :: File push error for file : {}, Bank {}, FileTransferType {}, Error : {}",
refundType,
fileName,
bankName,
bankReportEntity.getFileTransferType(),
e.getMessage(),
e
);
} finally {
log.info("{} :: SFTP to {} Status Updated for : {}", refundType, bankName, bankReportEntity);
bankReportEntity.setSftpStatus(sftpStatus);
bankReportEntity.setErrorReason(errorReason);
}
}
private SFTPPushFileCountDto getSFTPSuccessfulFileCount(
String bankName,
String path,
String refundType,
List<BankReportEntity> pushedFiles,
List<BankReportEntity> failedBankReports) {
int totalSuccessfulPush = pushedFiles.size();
int totalFailedPush = failedBankReports.size();
log.info("{} :: getSFTPSuccessfulFileCount() for {}, from {}", refundType, bankName, path);
try {
List<String> remoteFiles = gateway.executeCommand(path);
for (Iterator<BankReportEntity> pushedFilesIterator = pushedFiles.iterator(); pushedFilesIterator.hasNext(); ) {
BankReportEntity bankReport = pushedFilesIterator.next();
String fileName = bankReport.getFileName();
if (!remoteFiles.contains(fileName)) {
log.error("getSFTPSuccessfulFileCount() : File not found in remote {}. File: {}", path, fileName);
totalFailedPush++;
totalSuccessfulPush--;
bankReport.setSftpStatus(SlotBankStatus.BANK_SFTP_PENDING.name());
bankReport.setErrorReason(ErrorCode.UNKNOWN_ERROR_CODE.name());
pushedFilesIterator.remove();
failedBankReports.add(bankReport);
emailService.sendSFTPExceptionEmail(
"File push error for file : " + fileName +
" and FileTransferType " + bankReport.getFileTransferType() +
". Error : " + ErrorCode.UNKNOWN_ERROR_CODE.description(),
bankName);
}
}
} catch (Exception ex) {
emailService.sendSFTPExceptionEmail("SFTP file count Failed from path " + path, bankName);
log.error("{} :: getSFTPSuccessfulFileCount() Failed for {}. Error : {}",
refundType,
bankName,
ex.getMessage(),
ex);
}
return SFTPPushFileCountDto.builder()
.totalSuccessfulPush(totalSuccessfulPush)
.totalFailedPush(totalFailedPush)
.build();
}
We can't reproduce the problem in our environment.
Can anybody help?

Related

AWS S3 uploaded file was not shown on cyberduck

I am trying to write an API for uploading and downloading files. After I uploaded a test file, it was not shown on cyberduck, but I can download the test file that I just uploaded.
Then I try to download the file that exist on cyberduck, but it shows
com.emc.object.s3.S3Exception: The specified key does not exist.
API code:
StorageImpl.java
#Service
public class StorageImpl implements Storage {
private static Logger logger = LoggerFactory.getLogger(Storage.class);
#Value("${storage.file.repository}")
private String fileRepository;
#Value("${object.storage.user}")
private String oUser;
#Value("${object.storage.endpoint}")
private String endpoint;
#Value("${object.storage.bucket}")
private String nBucket;
#Value("${object.storage.key.secret}")
private String nSecret;
#Value("${object.storage.region.name}")
private String nRegion;
private S3Client s3client;
public StorageImpl() {
}
#Inject
void init() {
try {
s3client = this.getS3Client();
} catch (Exception e) {
e.printStackTrace();
}
}
private S3Client getS3Client() {
if (s3client == null) {
try {
SSLContext sc = SSLContext.getInstance("TLSv1.2");
sc.init(null, null, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(new PreferredCipherSuiteSSLSocketFactory(sc.getSocketFactory()));
String hostname = "";
URI uri = null;
try {
uri = new URI(endpoint);
hostname = uri.getHost();
} catch (URISyntaxException e) {
logger.error("URL " + endpoint + " is a malformed URL");
e.printStackTrace();
}
S3Config config = null;
config = new S3Config(new URI(endpoint)).withUseVHost(false);
logger.debug("oUser=" + oUser + ", secret=" + nSecret + ", endpoint=" + endpoint);
config.withIdentity(oUser).withSecretKey(nSecret);
logger.debug("");
config.setSignMetadataSearch(true);
s3client = new S3JerseyClient(config, new URLConnectionClientHandler());
logger.debug("s3client initiated. endpoint: " + endpoint);
} catch (Exception ex) {
logger.error(ex.getMessage(), ex);
}
}
return s3client;
}
private File convertMultiPartFileToFile(final MultipartFile multipartFile) {
final File file = new File(multipartFile.getOriginalFilename());
try (final FileOutputStream outputStream = new FileOutputStream(file)) {
outputStream.write(multipartFile.getBytes());
} catch (IOException e) {
logger.error("Error {} occurred while converting the multipart file", e.getLocalizedMessage());
}
return file;
}
#Override
public void save(final MultipartFile multipartFile) {
try {
final File file = convertMultiPartFileToFile(multipartFile);
logger.info("Uploading file with name {}", file.getName());
final PutObjectRequest putObjectRequest = new PutObjectRequest(nBucket, file.getName(), file);
s3client.putObject(putObjectRequest);
Files.delete(file.toPath()); // Remove the file locally created in the project folder
} catch (AmazonServiceException e) {
logger.error("Error {} occurred while uploading file", e.getLocalizedMessage());
} catch (IOException ex) {
logger.error("Error {} occurred while deleting temporary file", ex.getLocalizedMessage());
}catch(S3Exception ae){
ae.printStackTrace();
logger.error("", ae);
}
}
#Override
public InputStream retrieve(String fileName) {
return s3client.getObject(nBucket, fileName).getObject();
}
}
FileController.java
#RestController
#RequestMapping("/files")
#CrossOrigin(origins = "*", maxAge = 3600)
public class FileController {
private static final String MESSAGE_1 = "Uploaded the file successfully";
private static final String FILE_NAME = "fileName";
#Autowired
protected StorageImpl storageImpl;
#Autowired
protected FileService fileService;
#GetMapping
public ResponseEntity<Object> findByName(#RequestParam("fileName") String fileName) {
return ResponseEntity
.ok()
.cacheControl(CacheControl.noCache())
.header("Content-type", "application/octet-stream")
.header("Content-disposition", "attachment; filename=\"" + fileName + "\"")
.body(new InputStreamResource(storageImpl.retrieve(fileName)));
}
#PostMapping
public ResponseEntity<Object> save(#RequestParam("file") MultipartFile multipartFile) {
storageImpl.save(multipartFile);
return new ResponseEntity<>(MESSAGE_1, HttpStatus.OK);
}
}
What are the possible reasons causing this bug?

How to write a junit test case for private methods that are inside an if condition?

This is my service class, It has public method getFilesFromDirAndUploadtoHost() and inside that i have a reattempt() inside an if condition. I need to cover more codes by junit. What will be the modified test class, test class i mentioned below.*
#Service
public class FileTransferServiceImpl {
#Override
public Boolean getFilesFromDirAndUploadtoHost() throws SftpException {
List<String> files = Stream.of(new File(sourcePath).listFiles()).filter(file -> !file.isDirectory())
.map(File::getName).filter(file -> !file.endsWith("zip")).collect(Collectors.toList());
if(files.isEmpty()){
logger.info("No Files found to transfer");
return true;
}
ChannelSftp channelSftp = createChannelSftp();
Boolean result = false;
Integer attemptNo = 0;
Date date = new Date();
SimpleDateFormat formatDate = new SimpleDateFormat("dd-MM-yyyy HH mm ss z");
formatDate.setTimeZone(TimeZone.getTimeZone("IST"));
String dirName = formatDate.format(date);
if (channelSftp != null) {
// Create new folder in target
try {
channelSftp.mkdir("/" + dirName);
} catch (SftpException e) {
logger.error("Directory creation unsuccessful", e);
}
// Initial Attempt
List<ProcessFile> processFiles = copyFilestoTarget(channelSftp, files, attemptNo, dirName);
List<ProcessFile> failedFiles = processFiles.stream().filter(processFile -> !processFile.getSuccessfulYN())
.collect(Collectors.toList());
// If any Failed files found set to retry
if (!failedFiles.isEmpty()) {
reattempt(channelSftp, failedFiles, attemptNo, processFiles, dirName);
}
disconnectChannelSftp(channelSftp);
// populaate Process Run
populateAndSaveProcessRun(processFiles, files, dirName);
result = true;
}
return result;
}
private List<ProcessFile> copyFilestoTarget(ChannelSftp channelSftp, List<String> files, Integer attempNo,
String dirName) {
List<ProcessFile> processFiles = new ArrayList<>();
for (String fileName : files) {
try {
if (!channelSftp.isConnected()) {
channelSftp = recreateChannelSftp();
}
channelSftp.put(sourcePath + "/" + fileName, targetPath + "/" + dirName);
processFiles.add(populateProcessFiles(fileName, attempNo, dirName, true));
} catch (JSchException e) {
logger.error("Reconnection failed attempt No: " + attempNo);
processFiles.add(populateProcessFiles(fileName, attempNo, dirName, false));
} catch (SftpException e) {
logger.error("File : " + fileName + " Transfer Failed Attempt No: " + attempNo);
processFiles.add(populateProcessFiles(fileName, attempNo, dirName, false));
}
}
return processFiles;
}
private void reattempt(ChannelSftp channelSftp, List<ProcessFile> failedFiles, Integer attemptNo,
List<ProcessFile> processFiles, String dirName) {
attemptNo = 1;
List<ProcessFile> newFailedFile = failedFiles;
while (attemptNo <= 3) {
newFailedFile = copyFilestoTarget(channelSftp,
newFailedFile.stream().filter(processFile -> !processFile.getSuccessfulYN())
.map(processFile -> processFile.getFileName()).collect(Collectors.toList()),
attemptNo, dirName);
processFiles.addAll(newFailedFile);
if (newFailedFile.stream().filter(processFile -> !processFile.getSuccessfulYN())
.collect(Collectors.toList()).isEmpty()) {
attemptNo = 4;
} else {
attemptNo++;
}
}
}
}
This is my test class what i shoule modify here so that i can cover more lines. Here i am not sure how to do that. If any one can help it will be great.
#ExtendWith(MockitoExtension.class)
public class FileTransferServiceTest {
#InjectMocks
private FileTransferServiceImpl fileTransferService = new FileTransferServiceImpl();
#Mock
private ProcessFileService processFileService;
#BeforeEach
public void beforeClass() {
ReflectionTestUtils.setField(fileTransferService, "sourcePath", "F:\\Sample-sftp prjt\\needToTransfer");
ReflectionTestUtils.setField(fileTransferService, "host", "SDC-CDPGP01-test.com.au");
ReflectionTestUtils.setField(fileTransferService, "port", 2222);
ReflectionTestUtils.setField(fileTransferService, "username", "tester");
ReflectionTestUtils.setField(fileTransferService, "password", "password");
ReflectionTestUtils.setField(fileTransferService, "sessionTimeout", 15000);
ReflectionTestUtils.setField(fileTransferService, "channelTimeout", 15000);
ReflectionTestUtils.setField(fileTransferService, "targetPath", "/");
}
#Test
void uploadToAemo() throws SftpException {
assertNotNull(fileTransferService.getFilesFromDirAndUploadtoHost());
}
}

File upload with in Spring MVC without adding any additional parameter in controller method

I am using spring boot 2. My new task is file uploading. I already did it. But I am asked to do it without adding a additional parameter to controller method like #RequestParam("files") MultipartFile files[]. I want to get this from request instead of adding this parameter.
How can I solve this?
I am adding my current code following.
#RequestMapping(value="/uploadMultipleFiles", method=RequestMethod.POST)
public #ResponseBody String handleFileUpload( #RequestParam("files") MultipartFile files[]){
try {
String filePath="c:/temp/kk/";
StringBuffer result=new StringBuffer();
byte[] bytes=null;
result.append("Uploading of File(s) ");
for (int i=0;i<files.length;i++) {
if (!files[i].isEmpty()) {
bytes = files[i].getBytes();
BufferedOutputStream stream = new BufferedOutputStream(new FileOutputStream(new File(filePath+files[i].getOriginalFilename())));
stream.write(bytes);
stream.close();
result.append(files[i].getOriginalFilename() + " Ok. ") ;
}
else
result.append( files[i].getOriginalFilename() + " Failed. ");
}
return result.toString();
} catch (Exception e) {
return "Error Occured while uploading files." + " => " + e.getMessage();
}
}
You can get files from HttpRequest:
#RequestMapping(value="/uploadMultipleFiles", method=RequestMethod.POST)
public String handleFileUpload(HttpRequest request){
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
Map<String, MultipartFile> yourFiles = multipartRequest.getFileMap();
return "All is Ok!";
}
My sample code.
#RequestMapping(value = "/multiple/upload", method = RequestMethod.POST)
public #ResponseBody String test(#RequestParam(value = "files[]") List<MultipartFile> files,
HttpServletRequest req) {
MultipartFileWriter writer = new MultipartFileWriter();
String folderPath = "/file/";
for (MultipartFile file : files) {
writer.writeFile(file, folderPath, req);
}
return "success";
}

SSLException:Write error: ssl=0x7f8170c780: I/O error during system call, Connection timed out

I use okhttp and retrofit2 to post a audio file,but sometimes the file broken (server side),What is the reason?
File file = new File(path);
RequestBody requestFile = RequestBody.create(MediaType.parse("audio/*"), file);
body = MultipartBody.Part.createFormData("audio_file", file.getName(), requestFile);
HashMap<String,RequestBody> params = new HashMap<>();
params.put("lecture_id",RequestBody.create(MediaType.parse("multipart/form-data"), mId + ""));
params.put("duration",RequestBody.create(MediaType.parse("multipart/form-data"),attachment.getDuration() + ""));
params.put("reply_message_id",RequestBody.create(MediaType.parse("multipart/form-data"),msg.getReplyMsgId() + ""));
Subscription subscription = ApiService.getInstance().sendAudioMessage(body,params).subscribe(new HttpObserver<IMMessage>() {
#Override
protected void onError(ApiException ex) {
CrashReport.postCatchedException(ex);
ToastUtil.showToast(getActivity(),ex.getMsg());
onSendMessageFail(msg);
}
#Override
public void onNext(IMMessage imMessage) {
onSendMessageSuccess(msg);
}
});
mCompositeSubscription.add(subscription);

Spring error: IllegalStateException: Cannot call sendRedirect() after the response has been committed and getOutputStream() has already been called

I have got an error when I try to download a zipped file using Spring boot and spring MVC:
Errors:
SEVERE: Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is java.lang.IllegalStateException: Cannot call sendRedirect() after the response has been committed] with root cause
java.lang.IllegalStateException: Cannot call sendRedirect() after the response has been committed
SEVERE: Servlet.service() for servlet dispatcherServlet threw exception
java.lang.IllegalStateException: getOutputStream() has already been called for this response
Basically, my app is loading a file, zipping it and download the new zip file.
This is my application Controller:
#Controller
public class ApplicationController {
public String fileZipped;
public String inputFile;
#RequestMapping(method = RequestMethod.GET, value = "/uploadForm")
public String provideUploadInfo() {
return "uploadForm";
}
#RequestMapping(method = RequestMethod.POST, value = "/")
public String handleFileUpload(#RequestParam("file") MultipartFile file,
RedirectAttributes redirectAttributes,
HttpServletResponse downloadResponse) {
if (!file.isEmpty()) {
try {
inputFile = file.getOriginalFilename();
BufferedOutputStream stream = new BufferedOutputStream(
new FileOutputStream(new File(Application.UPLOAD_DIR + "/" + inputFile)));
FileCopyUtils.copy(file.getInputStream(), stream);
stream.close();
redirectAttributes.addFlashAttribute("message",
"You successfully uploaded " + file.getOriginalFilename() + "!");
FileUtils.copyDirectory(Application.UPLOAD_DIR, Application.OUTPUT_FOLDER);
FileUtils.cleanDirectory(new File(Application.UPLOAD_DIR));
}
catch (Exception e) {
redirectAttributes.addFlashAttribute("message",
"You failed to upload " + file.getOriginalFilename() + " => " + e.getMessage());
}
}
else {
redirectAttributes.addFlashAttribute("message",
"You failed to upload " + file.getOriginalFilename() + " because the file was empty");
}
return "redirect:/zipFiles";
}
#RequestMapping(value = "/download")
public String handleFileDownload(HttpServletResponse downloadResponse) throws IOException {
InputStream inputStream = null;
OutputStream outStream = null;
try {
if ( fileZipped!=null && Files.exists(Paths.get(fileZipped))) {
inputStream = new FileInputStream(fileZipped);
downloadResponse.setContentType(fileZipped);
downloadResponse.addHeader("Content-Disposition", "attachment; filename=" + Paths.get(fileZipped).getFileName().toString());
outStream = downloadResponse.getOutputStream();
org.apache.commons.io.IOUtils.copy(inputStream, outStream);
downloadResponse.flushBuffer();
}
} catch (IOException e) {
throw new RuntimeException("IOError writing file to output stream");
} finally {
if (inputStream != null) inputStream.close();
if (outStream != null) outStream.close();
}
return "redirect:/uploadForm";
}
#RequestMapping(value="/zipFiles")
public String handleFileZip() throws IOException {
if(inputFile!=null) {
fileZipped = zipFiles(Application.OUTPUT_FOLDER, inputFile);
}
return "redirect:/download";
}
private String zipFiles(String folder, String zipFileName) throws IOException {
String zipFile = folder + "/" + FilenameUtils.removeExtension(zipFileName) + ".zip";
FileOutputStream fileOutputstream = new FileOutputStream(zipFile);
ZipOutputStream zipOutputStream = new ZipOutputStream(new BufferedOutputStream(fileOutputstream));
File []filesArray = new File(folder).listFiles();
for (File file : filesArray){
if (!FilenameUtils.getExtension(file.getAbsolutePath()).equals("zip")) {
byte[] buffer = new byte[1024];
FileInputStream fileInputStream = new FileInputStream(file);
zipOutputStream.putNextEntry(new ZipEntry(file.getName()));
int length;
while ((length = fileInputStream.read()) > 0) {
zipOutputStream.write(buffer, 0, length);
}
zipOutputStream.closeEntry();
fileInputStream.close();
}
}
zipOutputStream.close();
return zipFile;
}
I do no know why that is happening if I am closing the input and output streams.
Thanks you very much for your help.
The problem is with the redirect. Instead of
return "redirect:/uploadForm";
return the view name.
return "/uploadForm";

Resources