Upload File data into database using spring mvc and hibernate - spring

I need to upload data into 2 database tables(factory and factoryType) using the below form but it isnt working can somebody take a look :
Factory table:factoryId,factoryname
FactoryType table: factoryType,factoryTypeId
FactoryConf table: factoryID,factoryTypeId
We are using hibernate for the database operations.
Model:
#Entity
#Table(name = "FactoryConf", uniqueConstraints = {
#UniqueConstraint(columnNames = { "factoryId" } )
})
public class FactoryConf {
#Id
long factoryId;
#OneToOne
#JoinColumn(name = "factoryId", insertable = false, updatable = false)
Factory factory;
#ManyToOne(optional = false)
#JoinColumn(name = "factoryTypeId")
FactoryType factoryType;
public FactoryConf() {
super();
}
public FactoryConf(long factoryId, FactoryType factoryType) {
super();
this.factoryType = factoryType;
this.factoryId = factoryId;
}
public Factory getFactory() {
return factory;
}
public void setFactory(Factory factory) {
this.factory = factory;
}
public FactoryType getFactoryType() {
return factoryType;
}
public void setFactoryType(FactoryType factoryType) {
this.factoryType = factoryType;
}
public long getFactoryId() {
return factoryId;
}
public void setFactoryId(long factoryId) {
this.factoryId = factoryId;
}
public FactoryType getFactoryTypeByFactoryID(long factoryId){
return factoryType;
}
}
Bean class:
/**
* This bean is defined to parse each record from the CSV file.
* All records are mapped to instances of this bean class.
*
*/
public class FactoryCSVFileInputBean {
private String upload_id;
private String file_name;
private byte file_data;
private long Id;
private String Name;
private String Type;
//getter setters
}
CSV parsing:
/**
* This class is defined for following actions
* 1. Validate the input CSV file format, header columns.
* 2. Parse the CSV file into a list of beans.
* 3. Validate input records with missing data and prepare a valid factory list to be processed. *
*/
public class FactoryCSVUtil {
private static Log log = LogFactory.getLog(FactoryCSVUtil.class);
private static final List<String> fileHeaderFields = new ArrayList<String>();
private static final String UTF8CHARSET = "UTF-8";
static{
for (Field f : FactoryCSVFileInputBean.class.getDeclaredFields()) {
fileHeaderFields.add(f.getName());
}
}
public static List<FactoryCSVFileInputBean> getCSVInputList(InputStream inputStream){
CSVReader reader = null;
List<FactoryCSVFileInputBean> csvList = null;
FactoryCSVFileInputBean inputRecord = null;
String[] header = null;
String[] row = null;
try {
reader = new CSVReader(new InputStreamReader(inputStream,UTF8CHARSET));
csvList = new ArrayList<FactoryCSVFileInputBean>();
header = reader.readNext();
boolean isEmptyLine = true;
while ((row = reader.readNext()) != null) {
isEmptyLine = true;
if(!(row.length==1 && StringUtils.isBlank(row[0]))){//not an empty line, not even containing ','
inputRecord = new FactoryCSVFileInputBean();
isEmptyLine = populateFields(inputRecord, header, row);
if(row.length != header.length)
//inputRecord.setUploadStatus("Not Loaded - Missing or invalid Data");
if(!isEmptyLine)
csvList.add(inputRecord);
}
}
} catch (IOException e) {
log.debug("IOException while accessing FactoryCSVFileInputBean: " + e);
return null;
} catch (IllegalAccessException e) {
log.debug("IllegalAccessException while accessing FactoryCSVFileInputBean: " + e);
return null;
} catch (InvocationTargetException e) {
log.debug("InvocationTargetException while copying FactoryCSVFileInputBean properties: " + e);
return null;
} catch (Exception e) {
log.debug("Exception while parsing CSV file: " + e);
return null;
}finally{
try{
if(reader!=null)
reader.close();
}catch(IOException ioe){}
}
return csvList;
}
protected static boolean populateFields(FactoryCSVFileInputBean inputRecord,String[] header, String[] row) throws IllegalAccessException, InvocationTargetException {
boolean isEmptyLine = true;
for (int i = 0; i < row.length; i++) {
String val = row[i];
if(!StringUtils.isBlank(val)){
BeanUtilsBean.getInstance().copyProperty(inputRecord, header[i], val);
isEmptyLine = false;
} else {
//inputRecord.setUploadStatus(String.format("Not Loaded - Missing or invalid Data for:%s",header[i]));
}
}
return isEmptyLine;
}
public static void validateInputFile(CommonsMultipartFile csvFile, Model model){
InputStream inputStream = null;
CSVReader reader = null;
String fileName = csvFile.getOriginalFilename();
String fileExtension = fileName.substring(fileName.lastIndexOf('.') + 1);
if(fileExtension.toUpperCase().equals("CSV")){
try{
inputStream = csvFile.getInputStream();
reader = new CSVReader(new InputStreamReader(inputStream,UTF8CHARSET));
String[] header = reader.readNext();
if(header!=null){
for (int i = 0; i < header.length; i++) {
if(!header[i].equals("") && !fileHeaderFields.contains(header[i])){
log.debug("Invalid Column found in upload file: " + header[i]);
model.addAttribute("failureMsg", "Invalid Column found in upload file: " + header[i]);
break;
}
}
for(csvHeaderFieldsEnum field : csvHeaderFieldsEnum.values()){
if(!Arrays.asList(header).contains(field.getValue())){
log.debug("Missing column in upload file: " + field.getValue());
model.addAttribute("failureMsg", "Missing column in upload file: " + field.getValue());
break;
}
}
}else{
model.addAttribute("failureMsg", "File is Empty - Please select a valid file");
}
String[] data = reader.readNext();
if(data==null){
log.debug("Empty file with header - No data found");
model.addAttribute("failureMsg", "Empty file with header - No data found");
}
}catch(IOException e){
log.debug("IOException in reading the CSV file: " + e);
model.addAttribute("failureMsg", "Exception in reading the CSV file");
}finally{
if(reader!=null)
try{
reader.close();
}catch(IOException e){ log.debug("IOException in closing reader of CSV file: " + e);}
}
}
else{
model.addAttribute("failureMsg", "Invalid file format - Please select a CSV file");
}
}
}
Model
public class FactoryUploadForm {
private CommonsMultipartFile fileData;
private String uploadComment;
/**
* #return the fileData
*/
public CommonsMultipartFile getFileData() {
return fileData;
}
/**
* #param fileData the fileData to set
*/
public void setFileData(CommonsMultipartFile fileData) {
this.fileData = fileData;
}
/**
* #return the uploadComment
*/
public String getUploadComment() {
return uploadComment;
}
/**
* #param uploadComment the uploadComment to set
*/
public void setUploadComment(String uploadComment) {
this.uploadComment = uploadComment;
}
public String toString(){
return " CSVFileName: " + getFileData().getOriginalFilename() + "; Upload Comment: " + uploadComment;
}
}
Controller
#Controller
public class FactoryUploadDownloadController {
private static final Log logger = LogFactory.getLog(FactoryUploadDownloadController.class);
#Resource
Service Service;
#Resource
FactoryUploadRepository repository;
#RequestMapping(value = "/submitUploadFactoryForm")
public String uploadFactory(FactoryUploadForm uploadform,
HttpServletRequest request, Model model, BindingResult result) {
logger.debug("====================================================================");
List<FactoryCSVFileInputBean> csvList = null;
List<FactoryType> factoryTypes = Service.getFactoryTypes();
try {
CommonsMultipartFile file = uploadform.getFileData();
// parse csv file to list
csvList = FactoryCSVUtil.getCSVInputList(file.getInputStream());
if (csvList == null) {
model.addAttribute("failureMsg","Error in file parsing - Please verify the file");
logger.debug("---------------------------------------------------------");
return "sucess";
}
} catch (Exception e) {
logger.debug("sorry this isn't working for you");
}
try {
CommonsMultipartFile file = uploadform.getFileData();
for (FactoryCSVFileInputBean inputRecord : csvList) {
Factory factoryentity = new Factory();
factoryentity.setId(inputRecord.getId());
factoryentity.setName(inputRecord.getName());
factoryentity = this.Service.saveFactory(factoryentity);
FactoryConf factoryconf = new FactoryConf();
factoryconf.setFactory(factoryentity);
factoryconf.setFactoryType(pickFactoryType(factoryTypes,inputRecord.getType()));
model.addAttribute("factoryconf", factoryconf);
this.Service.savefactoryCfg(factoryconf);
}
} catch (Exception e) {
logger.debug("sorry this isnt working for you");
}
return "success";
}
private FactoryType pickFactoryType(List<FactoryType> types, String typeName) {
for (FactoryType type : types) {
if (type.getFactoryType().equalsIgnoreCase(typeName))
return type;
}
throw new RuntimeException(String.format("Factory Type Invalid :%s", typeName));
}
}

From your question, I understand that you are not able to parse data from a CSV file. Here is sample code for similar task. I think it should help.

Related

How to write a junit test case for private methods that are inside an if condition?

This is my service class, It has public method getFilesFromDirAndUploadtoHost() and inside that i have a reattempt() inside an if condition. I need to cover more codes by junit. What will be the modified test class, test class i mentioned below.*
#Service
public class FileTransferServiceImpl {
#Override
public Boolean getFilesFromDirAndUploadtoHost() throws SftpException {
List<String> files = Stream.of(new File(sourcePath).listFiles()).filter(file -> !file.isDirectory())
.map(File::getName).filter(file -> !file.endsWith("zip")).collect(Collectors.toList());
if(files.isEmpty()){
logger.info("No Files found to transfer");
return true;
}
ChannelSftp channelSftp = createChannelSftp();
Boolean result = false;
Integer attemptNo = 0;
Date date = new Date();
SimpleDateFormat formatDate = new SimpleDateFormat("dd-MM-yyyy HH mm ss z");
formatDate.setTimeZone(TimeZone.getTimeZone("IST"));
String dirName = formatDate.format(date);
if (channelSftp != null) {
// Create new folder in target
try {
channelSftp.mkdir("/" + dirName);
} catch (SftpException e) {
logger.error("Directory creation unsuccessful", e);
}
// Initial Attempt
List<ProcessFile> processFiles = copyFilestoTarget(channelSftp, files, attemptNo, dirName);
List<ProcessFile> failedFiles = processFiles.stream().filter(processFile -> !processFile.getSuccessfulYN())
.collect(Collectors.toList());
// If any Failed files found set to retry
if (!failedFiles.isEmpty()) {
reattempt(channelSftp, failedFiles, attemptNo, processFiles, dirName);
}
disconnectChannelSftp(channelSftp);
// populaate Process Run
populateAndSaveProcessRun(processFiles, files, dirName);
result = true;
}
return result;
}
private List<ProcessFile> copyFilestoTarget(ChannelSftp channelSftp, List<String> files, Integer attempNo,
String dirName) {
List<ProcessFile> processFiles = new ArrayList<>();
for (String fileName : files) {
try {
if (!channelSftp.isConnected()) {
channelSftp = recreateChannelSftp();
}
channelSftp.put(sourcePath + "/" + fileName, targetPath + "/" + dirName);
processFiles.add(populateProcessFiles(fileName, attempNo, dirName, true));
} catch (JSchException e) {
logger.error("Reconnection failed attempt No: " + attempNo);
processFiles.add(populateProcessFiles(fileName, attempNo, dirName, false));
} catch (SftpException e) {
logger.error("File : " + fileName + " Transfer Failed Attempt No: " + attempNo);
processFiles.add(populateProcessFiles(fileName, attempNo, dirName, false));
}
}
return processFiles;
}
private void reattempt(ChannelSftp channelSftp, List<ProcessFile> failedFiles, Integer attemptNo,
List<ProcessFile> processFiles, String dirName) {
attemptNo = 1;
List<ProcessFile> newFailedFile = failedFiles;
while (attemptNo <= 3) {
newFailedFile = copyFilestoTarget(channelSftp,
newFailedFile.stream().filter(processFile -> !processFile.getSuccessfulYN())
.map(processFile -> processFile.getFileName()).collect(Collectors.toList()),
attemptNo, dirName);
processFiles.addAll(newFailedFile);
if (newFailedFile.stream().filter(processFile -> !processFile.getSuccessfulYN())
.collect(Collectors.toList()).isEmpty()) {
attemptNo = 4;
} else {
attemptNo++;
}
}
}
}
This is my test class what i shoule modify here so that i can cover more lines. Here i am not sure how to do that. If any one can help it will be great.
#ExtendWith(MockitoExtension.class)
public class FileTransferServiceTest {
#InjectMocks
private FileTransferServiceImpl fileTransferService = new FileTransferServiceImpl();
#Mock
private ProcessFileService processFileService;
#BeforeEach
public void beforeClass() {
ReflectionTestUtils.setField(fileTransferService, "sourcePath", "F:\\Sample-sftp prjt\\needToTransfer");
ReflectionTestUtils.setField(fileTransferService, "host", "SDC-CDPGP01-test.com.au");
ReflectionTestUtils.setField(fileTransferService, "port", 2222);
ReflectionTestUtils.setField(fileTransferService, "username", "tester");
ReflectionTestUtils.setField(fileTransferService, "password", "password");
ReflectionTestUtils.setField(fileTransferService, "sessionTimeout", 15000);
ReflectionTestUtils.setField(fileTransferService, "channelTimeout", 15000);
ReflectionTestUtils.setField(fileTransferService, "targetPath", "/");
}
#Test
void uploadToAemo() throws SftpException {
assertNotNull(fileTransferService.getFilesFromDirAndUploadtoHost());
}
}

How to export huge result set from database into several csv files and zip them on the fly?

I need to create a REST controller which extracts data from a database and write it into CSV files that will ultimately be zipped together. Each CSV file should contain exactly 10 lines. Eventually all CSV files should be zipped into a one zip file. I want everything to happen on the fly, meaning - saving files to a temporary location on the disk is not an option. Can someone provide me with an example?
I found a very nice code to export huge amount of rows from database into several csv files and zip it.
I think this is a nice code that can assist alot of developers.
I have tested the solution and you can find the entire example at : https://github.com/idaamit/stream-from-db/tree/master
The conroller is :
#GetMapping(value = "/employees/{employeeId}/cars") #ResponseStatus(HttpStatus.OK) public ResponseEntity<StreamingResponseBody> getEmployeeCars(#PathVariable int employeeId) {
log.info("Going to export cars for employee {}", employeeId);
String zipFileName = "Cars Of Employee - " + employeeId;
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_TYPE, "application/zip")
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + zipFileName + ".zip")
.body(
employee.getCars(dataSource, employeeId));
The employee class, first checks if we need to prepare more than one csv or not :
public class Employee {
public StreamingResponseBody getCars(BasicDataSource dataSource, int employeeId) {
StreamingResponseBody streamingResponseBody = new StreamingResponseBody() {
#Override
public void writeTo(OutputStream outputStream) throws IOException {
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
String sqlQuery = "SELECT [Id], [employeeId], [type], [text1] " +
"FROM Cars " +
"WHERE EmployeeID=? ";
PreparedStatementSetter preparedStatementSetter = new PreparedStatementSetter() {
public void setValues(PreparedStatement preparedStatement) throws SQLException {
preparedStatement.setInt(1, employeeId);
}
};
StreamingZipResultSetExtractor zipExtractor = new StreamingZipResultSetExtractor(outputStream, employeeId, isMoreThanOneFile(jdbcTemplate, employeeId));
Integer numberOfInteractionsSent = jdbcTemplate.query(sqlQuery, preparedStatementSetter, zipExtractor);
}
};
return streamingResponseBody;
}
private boolean isMoreThanOneFile(JdbcTemplate jdbcTemplate, int employeeId) {
Integer numberOfCars = getCount(jdbcTemplate, employeeId);
return numberOfCars >= StreamingZipResultSetExtractor.MAX_ROWS_IN_CSV;
}
private Integer getCount(JdbcTemplate jdbcTemplate, int employeeId) {
String sqlQuery = "SELECT count([Id]) " +
"FROM Cars " +
"WHERE EmployeeID=? ";
return jdbcTemplate.queryForObject(sqlQuery, new Object[] { employeeId }, Integer.class);
}
}
This class StreamingZipResultSetExtractor is responsible to split the csv streaming data into several files and zip it.
#Slf4j
public class StreamingZipResultSetExtractor implements ResultSetExtractor<Integer> {
private final static int CHUNK_SIZE = 100000;
public final static int MAX_ROWS_IN_CSV = 10;
private OutputStream outputStream;
private int employeeId;
private StreamingCsvResultSetExtractor streamingCsvResultSetExtractor;
private boolean isInteractionCountExceedsLimit;
private int fileCount = 0;
public StreamingZipResultSetExtractor(OutputStream outputStream, int employeeId, boolean isInteractionCountExceedsLimit) {
this.outputStream = outputStream;
this.employeeId = employeeId;
this.streamingCsvResultSetExtractor = new StreamingCsvResultSetExtractor(employeeId);
this.isInteractionCountExceedsLimit = isInteractionCountExceedsLimit;
}
#Override
#SneakyThrows
public Integer extractData(ResultSet resultSet) throws DataAccessException {
log.info("Creating thread to extract data as zip file for employeeId {}", employeeId);
int lineCount = 1; //+1 for header row
try (PipedOutputStream internalOutputStream = streamingCsvResultSetExtractor.extractData(resultSet);
PipedInputStream InputStream = new PipedInputStream(internalOutputStream);
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(InputStream))) {
String currentLine;
String header = bufferedReader.readLine() + "\n";
try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) {
createFile(employeeId, zipOutputStream, header);
while ((currentLine = bufferedReader.readLine()) != null) {
if (lineCount % MAX_ROWS_IN_CSV == 0) {
zipOutputStream.closeEntry();
createFile(employeeId, zipOutputStream, header);
lineCount++;
}
lineCount++;
currentLine += "\n";
zipOutputStream.write(currentLine.getBytes());
if (lineCount % CHUNK_SIZE == 0) {
zipOutputStream.flush();
}
}
}
} catch (IOException e) {
log.error("Task {} could not zip search results", employeeId, e);
}
log.info("Finished zipping all lines to {} file\\s - total of {} lines of data for task {}", fileCount, lineCount - fileCount, employeeId);
return lineCount;
}
private void createFile(int employeeId, ZipOutputStream zipOutputStream, String header) {
String fileName = "Cars for Employee - " + employeeId;
if (isInteractionCountExceedsLimit) {
fileCount++;
fileName += " Part " + fileCount;
}
try {
zipOutputStream.putNextEntry(new ZipEntry(fileName + ".csv"));
zipOutputStream.write(header.getBytes());
} catch (IOException e) {
log.error("Could not create new zip entry for task {} ", employeeId, e);
}
}
}
The class StreamingCsvResultSetExtractor is responsible for transfer the data from the resultset into csv file. There is more work to do to handle special character set which are problematic in csv cell.
#Slf4j
public class StreamingCsvResultSetExtractor implements ResultSetExtractor<PipedOutputStream> {
private final static int CHUNK_SIZE = 100000;
private PipedOutputStream pipedOutputStream;
private final int employeeId;
public StreamingCsvResultSetExtractor(int employeeId) {
this.employeeId = employeeId;
}
#SneakyThrows
#Override
public PipedOutputStream extractData(ResultSet resultSet) throws DataAccessException {
log.info("Creating thread to extract data as csv and save to file for task {}", employeeId);
this.pipedOutputStream = new PipedOutputStream();
ExecutorService executor = Executors.newSingleThreadExecutor();
executor.submit(() -> {
prepareCsv(resultSet);
});
return pipedOutputStream;
}
#SneakyThrows
private Integer prepareCsv(ResultSet resultSet) {
int interactionsSent = 1;
log.info("starting to extract data to csv lines");
streamHeaders(resultSet.getMetaData());
StringBuilder csvRowBuilder = new StringBuilder();
try {
int columnCount = resultSet.getMetaData().getColumnCount();
while (resultSet.next()) {
for (int i = 1; i < columnCount + 1; i++) {
if(resultSet.getString(i) != null && resultSet.getString(i).contains(",")){
String strToAppend = "\"" + resultSet.getString(i) + "\"";
csvRowBuilder.append(strToAppend);
} else {
csvRowBuilder.append(resultSet.getString(i));
}
csvRowBuilder.append(",");
}
int rowLength = csvRowBuilder.length();
csvRowBuilder.replace(rowLength - 1, rowLength, "\n");
pipedOutputStream.write(csvRowBuilder.toString().getBytes());
interactionsSent++;
csvRowBuilder.setLength(0);
if (interactionsSent % CHUNK_SIZE == 0) {
pipedOutputStream.flush();
}
}
} finally {
pipedOutputStream.flush();
pipedOutputStream.close();
}
log.debug("Created all csv lines for Task {} - total of {} rows", employeeId, interactionsSent);
return interactionsSent;
}
#SneakyThrows
private void streamHeaders(ResultSetMetaData resultSetMetaData) {
StringBuilder headersCsvBuilder = new StringBuilder();
for (int i = 1; i < resultSetMetaData.getColumnCount() + 1; i++) {
headersCsvBuilder.append(resultSetMetaData.getColumnLabel(i)).append(",");
}
int rowLength = headersCsvBuilder.length();
headersCsvBuilder.replace(rowLength - 1, rowLength, "\n");
pipedOutputStream.write(headersCsvBuilder.toString().getBytes());
}
}
In order to test this, you need to execute http://localhost:8080/stream-demo/employees/3/cars

How to change targetSdkVersion

I am having an issue with target targetSdkVersion.I want to move from 27 to 29 but when change to sdk 29 the app runs but does not display videos.But it displays the videos when it is sdk 27 or less.
Please can someone help me fix this.
/**
* A simple {#link Fragment} subclass.
*/
public class ChannelFragment extends Fragment {
private static String GOOGLE_YOUTUBE_API_KEY = "AIzaSyBJQYpQRTzM5wuuhMUxmP7rvP3lbMGtUZ8";//here you should use your api key for testing purpose you can use this api also
private static String CHANNEL_ID = "UCB_ZwuWCAuB7y0B93qvnkWw"; //here you should use your channel id for testing purpose you can use this api also
private static String CHANNLE_GET_URL = "https://www.googleapis.com/youtube/v3/search?part=snippet&order=date&channelId=" + CHANNEL_ID + "&maxResults=50&key=" + GOOGLE_YOUTUBE_API_KEY + "";
private RecyclerView mList_videos = null;
private VideoPostAdapter adapter = null;
private ArrayList<YoutubeDataModel> mListData = new ArrayList<>();
public ChannelFragment() {
// Required empty public constructor
}
#Override
public View onCreateView(LayoutInflater inflater, ViewGroup container,
Bundle savedInstanceState) {
// Inflate the layout for this fragment
View view = inflater.inflate(R.layout.fragment_channel, container, false);
mList_videos = (RecyclerView) view.findViewById(R.id.mList_videos);
initList(mListData);
new RequestYoutubeAPI().execute();
return view;
}
private void initList(ArrayList<YoutubeDataModel> mListData) {
mList_videos.setLayoutManager(new LinearLayoutManager(getActivity()));
adapter = new VideoPostAdapter(getActivity(), mListData, new OnItemClickListener() {
#Override
public void onItemClick(YoutubeDataModel item) {
YoutubeDataModel youtubeDataModel = item;
Intent intent = new Intent(getActivity(), DetailsActivity.class);
intent.putExtra(YoutubeDataModel.class.toString(), youtubeDataModel);
startActivity(intent);
}
});
mList_videos.setAdapter(adapter);
}
//create an asynctask to get all the data from youtube
private class RequestYoutubeAPI extends AsyncTask<Void, String, String> {
#Override
protected void onPreExecute() {
super.onPreExecute();
}
#Override
protected String doInBackground(Void... params) {
DefaultHttpClient httpClient = new DefaultHttpClient();
HttpGet httpGet = new HttpGet(CHANNLE_GET_URL);
Log.e("URL", CHANNLE_GET_URL);
try {
HttpResponse response = httpClient.execute(httpGet);
HttpEntity httpEntity = response.getEntity();
String json = EntityUtils.toString(httpEntity);
return json;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
#Override
protected void onPostExecute(String response) {
super.onPostExecute(response);
if (response != null) {
try {
JSONObject jsonObject = new JSONObject(response);
Log.e("response", jsonObject.toString());
mListData = parseVideoListFromResponse(jsonObject);
initList(mListData);
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
public ArrayList<YoutubeDataModel> parseVideoListFromResponse(JSONObject jsonObject) {
ArrayList<YoutubeDataModel> mList = new ArrayList<>();
if (jsonObject.has("items")) {
try {
JSONArray jsonArray = jsonObject.getJSONArray("items");
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject json = jsonArray.getJSONObject(i);
if (json.has("id")) {
JSONObject jsonID = json.getJSONObject("id");
String video_id = "";
if (jsonID.has("videoId")) {
video_id = jsonID.getString("videoId");
}
if (jsonID.has("kind")) {
if (jsonID.getString("kind").equals("youtube#video")) {
YoutubeDataModel youtubeObject = new YoutubeDataModel();
JSONObject jsonSnippet = json.getJSONObject("snippet");
String title = jsonSnippet.getString("title");
String description = jsonSnippet.getString("description");
String publishedAt = jsonSnippet.getString("publishedAt");
String thumbnail = jsonSnippet.getJSONObject("thumbnails").getJSONObject("high").getString("url");
youtubeObject.setTitle(title);
youtubeObject.setDescription(description);
youtubeObject.setPublishedAt(publishedAt);
youtubeObject.setThumbnail(thumbnail);
youtubeObject.setVideo_id(video_id);
mList.add(youtubeObject);
}
}
}
}
} catch (JSONException e) {
e.printStackTrace();
}
}
return mList;
}
}

How to import data from csv to elasticsearch in java without using logstash?

I want to import data from a csv file to elasticsearch. But I don't want to use logstatsh. So, what are the ways I can do this ?
Any blogs ? Docs ?
I came across TransportClient, but I'm not getting the point from where to start .
Thanks in advance.
very late answer, however :) ….This is for elasticsearch 7.6.0
//this class for keeping csv each row values
public class Document {
private String id;
private String documentName;
private String name;
private String title;
private String dob;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getDocumentName() {
return documentName;
}
public void setDocumentName(String documentName) {
this.documentName = documentName;
}
public String getName() {
return name1;
}
public void setName(String name1) {
this.name1 = name1;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getDob() {
return dob;
}
public void setDob(String dob) {
this.dob = dob;
}
}
public void bulkInsert() {
long starttime = System.currentTimeMillis();
logger.debug("ElasticSearchServiceImpl => bulkInsert Service Started");
BufferedReader br = null;
String line = "";
String cvsSplitBy = ",";
BulkRequest request;
Document document;
//elastic Search Index Name
String esIndex = "post";
try {
br = new BufferedReader(new FileReader(<path to CSV>));
request = new BulkRequest();
while ((line = br.readLine()) != null) {
// use comma as separator
String[] row = line.split(cvsSplitBy);
if(row.length >= 1) {
//filling Document object using csv columns array
document = getDocEntity(row);
//adding each filled obect into BulkRequest
request.add(getIndexRequest(document, esIndex));
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : null row ="+row.toString());
}
}
br.close();
if(request.numberOfActions()>0) {
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if(bulkResponse.hasFailures()) {
logger.error("ElasticSearchServiceImpl => bulkInsert : Some of the record has failed.Please reinitiate the process");
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : Success");
}
} else {
logger.info("ElasticSearchServiceImpl => bulkInsert : No request for BulkInsert ="+request.numberOfActions());
}
} catch (Exception e) {
logger.error("ElasticSearchServiceImpl => bulkInsert : Exception =" + e.getMessage());
}
long endTime = System.currentTimeMillis();
logger.info("ElasticSearchServiceImpl => bulkInsert End" + Util.DB_AVG_RESP_LOG + "" + (endTime - starttime));
}
public static Document getDocEntity(String[] row)throws Exception {
Document document = new Document();
document.setId(UUID.randomUUID().toString());
for(int i=0;i<row.length;i++) {
switch (i) {
case 0:
document.setDocumentName(row[i]);
break;
case 1:
document.setName(row[i]);
break;
case 7:
document.setTitle(row[i]);
break;
case 8:
document.setDob(row[i]);
break;
}
return document;
}
public static IndexRequest getIndexRequest(Document document,String index)throws Exception {
IndexRequest indexRequest = null;
Map<String, Object> jsonMap = new HashMap<>();
jsonMap.put("doc_name",document.getDocumentName());
jsonMap.put("title",document.getTitle());
jsonMap.put("dob",document.getDob());
indexRequest = new IndexRequest(index).id(document.getId()).source(jsonMap);
return indexRequest;
}
If you need to show each response, you can use the following code for responses
for (BulkItemResponse bulkItemResponse : bulkResponse) {
DocWriteResponse itemResponse = bulkItemResponse.getResponse();
switch (bulkItemResponse.getOpType()) {
case INDEX:
case CREATE:
IndexResponse indexResponse = (IndexResponse) itemResponse;
break;
}
}
For more information please read official link

How to update/refresh list items of list view in oracle maf?

I have list of tasks. When I delete my one of the task my list still shows that task in the list but on server side i have updated list I am not able to refresh my list. I am getting new task array from server but not able to show it. When I launched my app again then it is showing the updated list. How can I get updated list without killing the app? Both the times I have updated array but not able to show it on the view.
public class ModelClass {
private String message;
private String statusCode;
private Response[] res = null;
protected transient PropertyChangeSupport _propertyChangeSupport = new PropertyChangeSupport(this);
public ModelClass() {
super();
clickEvent(new ActionEvent());
}
public static String taskId;
public void setTaskId(String taskId) {
System.out.print(taskId);
this.taskId = taskId;
}
public String getTaskId() {
return taskId;
}
public PropertyChangeSupport getPropertyChangeSupport() {
return _propertyChangeSupport;
}
public void setMessage(String message) {
String oldMessage = this.message;
this.message = message;
_propertyChangeSupport.firePropertyChange("message", oldMessage, message);
}
public String getMessage() {
return message;
}
public void setStatusCode(String statusCode) {
String oldStatusCode = this.statusCode;
this.statusCode = statusCode;
_propertyChangeSupport.firePropertyChange("statusCode", oldStatusCode, statusCode);
}
public String getStatusCode() {
return statusCode;
}
public void setRes(Response[] res) {
Response[] oldRes = this.res;
this.res = res;
System.out.println(res);
_propertyChangeSupport.firePropertyChange("res", oldRes, res);
System.out.println("refreshing here ");
}
public Response[] getRes() {
return res;
}
#Override
public String toString() {
return "ClassPojo [response = " + res + ", message = " + message + ", statusCode = " + statusCode + "]";
}
public void addPropertyChangeListener(PropertyChangeListener l) {
_propertyChangeSupport.addPropertyChangeListener(l);
}
public void removePropertyChangeListener(PropertyChangeListener l) {
_propertyChangeSupport.removePropertyChangeListener(l);
}
public class Response {
private String taskId;
private String taskType;
private Integer taskTime;
private String taskName;
private PropertyChangeSupport _propertyChangeSupport = new PropertyChangeSupport(this);
Response(String taskId, String taskType, String taskName) {
super();
this.taskId = taskId;
this.taskType = taskType;
this.taskName = taskName;
}
public void setTaskId(String taskId) {
String oldTaskId = this.taskId;
this.taskId = taskId;
_propertyChangeSupport.firePropertyChange("taskId", oldTaskId, taskId);
}
public String getTaskId() {
return taskId;
}
public void setTaskType(String taskType) {
String oldTaskType = this.taskType;
this.taskType = taskType;
_propertyChangeSupport.firePropertyChange("taskType", oldTaskType, taskType);
}
public String getTaskType() {
return taskType;
}
public void setTaskTime(Integer taskTime) {
Integer oldTaskTime = this.taskTime;
this.taskTime = taskTime;
_propertyChangeSupport.firePropertyChange("taskTime", oldTaskTime, taskTime);
}
public Integer getTaskTime() {
return taskTime;
}
public void setTaskName(String taskName) {
String oldTaskName = this.taskName;
this.taskName = taskName;
_propertyChangeSupport.firePropertyChange("taskName", oldTaskName, taskName);
}
public String getTaskName() {
return taskName;
}
#Override
public String toString() {
return "ClassPojo [taskId = " + taskId + ", taskType = " + taskType + ", taskTime = " + taskTime +
", taskName = " + taskName + "]";
}
public void addPropertyChangeListener(PropertyChangeListener l) {
_propertyChangeSupport.addPropertyChangeListener(l);
}
public void removePropertyChangeListener(PropertyChangeListener l) {
_propertyChangeSupport.removePropertyChangeListener(l);
}
}
protected transient ProviderChangeSupport providerChangeSupport = new ProviderChangeSupport(this);
public void addProviderChangeListener(ProviderChangeListener l) {
providerChangeSupport.addProviderChangeListener(l);
}
public void removeProviderChangeListener(ProviderChangeListener l) {
providerChangeSupport.removeProviderChangeListener(l);
}
public void clickEvent(ActionEvent actionEvent) {
try {
JSONObject paramsMap = new JSONObject();
paramsMap.put("userId", "1");
HttpURLConnection httpURLConnection = null;
try {
URL url = new URL("http://ec2-54-226-57-153.compute-1.amazonaws.com:8080/#########");
httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setReadTimeout(120000);
httpURLConnection.setConnectTimeout(120000);
httpURLConnection.setRequestMethod("POST");
httpURLConnection.setRequestProperty("Content-Type", "application/json");
httpURLConnection.setDoInput(true);
httpURLConnection.setDoOutput(true);
httpURLConnection.connect();
OutputStream os = httpURLConnection.getOutputStream();
BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
os.write(paramsMap.toString().getBytes());
bufferedWriter.flush();
bufferedWriter.close();
os.close();
if (httpURLConnection.getResponseCode() == HttpURLConnection.HTTP_OK) {
InputStream is = httpURLConnection.getInputStream();
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
StringBuilder builder = new StringBuilder();
String line = bufferedReader.readLine();
while (line != null) {
builder.append(line + "\n");
line = bufferedReader.readLine();
}
is.close();
if (httpURLConnection != null)
httpURLConnection.disconnect();
System.out.println(builder.toString());
JSONObject json = new JSONObject(builder.toString());
String status = json.optString("statusCode");
String message = json.optString("message");
String response = json.optString("response");
System.out.println(status);
System.out.println(message);
// System.out.println(response);
JSONArray objarr = json.optJSONArray("response");
Response[] temp_res = new Response[objarr.length()];
for (int i = 0; i < objarr.length(); i++) {
System.out.println(objarr.getJSONObject(i));
JSONObject obj = objarr.getJSONObject(i);
String task = obj.optString("taskName");
taskId = obj.optString("taskId");
String taskType = obj.optString("taskType");
System.out.println(task);
System.out.println(taskId);
System.out.println(taskType);
temp_res[i] = new Response(taskId, taskType, task);
}
setRes(temp_res);
} else {
if (httpURLConnection != null)
httpURLConnection.disconnect();
System.out.println("Invalid response from the server");
}
} catch (Exception e) {
e.printStackTrace();
if (httpURLConnection != null)
httpURLConnection.disconnect();
} finally {
if (httpURLConnection != null)
httpURLConnection.disconnect();
}
} catch (Exception e) {
e.printStackTrace();
}
}
I think you need to add providerChangeSupport.fireProviderRefresh("res");
and you have to make public method for providerChangeSupport.
Here is the link : https://community.oracle.com/message/13203364#13203364
Other than the fix suggested by Rashi Verma, there is one more change required.
The code snippet:
res = (Response[]) res_new.toArray(new Response[res_new.size()]);
setRes(res);
needs to be changed to:
Response[] temp_res;
temp_res = (Response[]) res_new.toArray(new
Response[res_new.size()]);
setRes(temp_res);
Currently, because the value of res is changed before invoking setRes, the propertyChangeEvent is not fired inside setRes. Both this propertyChangeEvent and the providerRefresh are needed for the changes you are making to start reflecting on the UI.

Resources