How to use accesslog_parser.pl Kannel access log parser? - kannel

I have found kannel access log parser accesslog_parser.pl in utils folder.
How to use this file for parse.
Also i want convert kannel_access.log in csv format. Please help me.

Assuming Kannel log is at /var/log/kannel/access.log
This should do
cat /var/log/kannel/access.log |accesslog_parser.pl

May be an old question but I'd like to share this small tool a wrote in java for parsing kannel log files and outputing the results in csv format:
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.StringTokenizer;
/**
*
* #author bashizip
*/
public class KannelLogsParser {
static String fileName;
/**
* #param args the command line arguments
*/
public static void main(String[] args) throws IOException {
// fileName = args[0];
fileName = "access_kannel.txt";
String allLines = readLines(fileName);
parseAndWriteToCsv(allLines);
}
static String readLines(String fileName) {
BufferedReader br = null;
String sCurrentLine = null;
StringBuilder sb = new StringBuilder();
try {
br = new BufferedReader(
new FileReader(fileName));
while ((sCurrentLine = br.readLine()) != null) {
sb.append(sCurrentLine).append("\n");
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (br != null) {
br.close();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
return sb.toString();
}
private static void parseAndWriteToCsv(String allLines) throws IOException {
String[] lines = allLines.split("\n");
System.out.println("lines to parse : " + lines.length);
StringTokenizer st;
StringBuilder output = new StringBuilder();
output.append("DATE;Heure;sent;SMS; SMSC; SVC; ACT; BINF; FID; from; to; flags; msg;;udh"+"\n");
int count = 0;
for (String line : lines) {
count++;
System.out.println("Parsing ..." + 100 * count / lines.length + "%");
st = new StringTokenizer(line, " ");
while (st.hasMoreTokens()) {
String currentToken = st.nextToken();
boolean messageToken = false;
boolean afterMessageToken = false;
if (currentToken.startsWith("[")) {
System.out.println(currentToken);
messageToken = currentToken.startsWith("[msg");
try {
currentToken = currentToken.substring(currentToken.indexOf(":") + 1, currentToken.indexOf("]"));
} catch (Exception e) {
System.err.println(e.getMessage());
messageToken = true;
currentToken = currentToken.substring(currentToken.indexOf(":") + 1);
}
}
currentToken = currentToken.replace("[", "");
currentToken = currentToken.replace("]", "");
output.append(currentToken);
if (!messageToken) {
output.append(";");
} else if (afterMessageToken) {
afterMessageToken = false;
output.append(" ");
} else {
output.append(" ");
afterMessageToken = true;
}
}
output.append("\n");
}
Files.write(Paths.get(fileName + ".csv"), output.toString().getBytes());
System.out.println("Output CVS file: " + fileName + ".csv");
System.out.println("Finished !");
}
}
Hope it can help someone one day :-)

Related

How to sanitize request object in the #restcontroller

I'm doing a static code analysis using checkmarx, which gave me medium vulnerabilities for #RequestBody.
Will place a sample code below, could someone help me fixing this issue.
#RestController
#RequestMapping("/api")
#Slf4j
#Validated
public class Myclass {
#Autowired
SplitClass split;
#PostMapping(path = "/something", consumes = "application/json", produces = "application/json")
public int splitter(#RequestBody SplitRequestVO **splitReq**) {
int response = 0;
try {
int value = split.methodName(splitReq);
if(value == 1) {
response = 1;
}else {
response = 0;
}
}catch(Exception e) {
log.error("Excpetion in the MY Class controller" +e.getMessage());
}
return response;
}
}
Every time it is showing error with this util class. please check whats wrong in this class.
#Slf4j
public class FileUtil {
private FileUtil() {
throw new IllegalArgumentException("FileUtil.class");
}
public static void createFileFromBytes(String fileName, byte[] bytes) throws IOException {
File file = new File(fileName);
try (FileOutputStream fos = new FileOutputStream(file)) {
fos.write(bytes);
fos.flush();
} catch (IOException exc) {
log.error("Exception in creating file from bytes : "+exc.getMessage());
}
}
public static void deleteFile(String fileName) {
log.info("Entry into deleteFile method.");
try {
File file = new File(fileName);
if (file.exists()) {
Files.delete(file.toPath());
if (log.isDebugEnabled())
log.debug("file deleted:" + fileName);
}
} catch (Exception e) {
log.error("Exception in deleting the file : "+e.getMessage());
}
log.info("Exit from deleteFile method ");
}
public static byte[] getBytesFromFile(File file) throws IOException {
log.info("Entry into getBytesFromFile method");
byte[] bytes = null;
try (InputStream is = new FileInputStream(file)) {
long length = file.length();
if (length > 2147483647L) {
log.error("File Too large:" + file.getName());
}
bytes = new byte[(int) length];
int offset = 0;
int numRead = 0;
while (offset < bytes.length && (numRead = is.read(bytes, offset, bytes.length - offset)) >= 0)
offset += numRead;
if (offset < bytes.length)
throw new IOException("Could not completely read file " + file.getName());
} catch (Exception e) {
log.error("Exception in getBytesFromFile : "+e.getMessage());
}
log.info("Exit from getBytesFromFile method ");
return bytes;
}
public static byte[] getBytesFromFile(String fileName) throws IOException {
File file = new File(fileName);
return getBytesFromFile(file);
}
}
Here im facing the vulnerability issue to fix in the block letters. Appreciated foy help.
Thanks in advance.

Run ansible playbook from java

Is there a way to execute ansible playbook from java.Can you please help me with reference could not find any good tutorial.
The simplest way is to run your command using ProcessBuilder:
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
public class AnsibleServive {
public static void main(String[] args) {
runCommand("ansible-playbook test.yml -i host.ini");
}
private static void runCommand(String command) {
ProcessBuilder processBuilder = new ProcessBuilder();
processBuilder.command("bash", "-c", command);
try {
Process process = processBuilder.start();
StringBuilder output = new StringBuilder();
StringBuilder errOutput = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
output.append(line + "\n");
}
BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
while ((line = errReader.readLine()) != null) {
errOutput.append(line + "\n");
}
int exitVal = process.waitFor();
if (exitVal == 0) {
System.out.println("Command Successfully executed.");
System.out.println(output);
} else {
System.err.println("Error ocured during running command.");
System.out.println(output);
System.err.println(errOutput);
}
} catch (IOException | InterruptedException e) {
System.err.println("Failed to execute command.");
e.printStackTrace();
}
}
}

Oracle : Send latest insert/update to JMS

I have an insert/update trigger for a Oracle table.
Is there a way to send the details of the affected row(all columns) as a message to JMS?
I can write a Java Program, 'loadjava' that and call from the trigger.
Does this way affect performance?
Is there any native way of achieving this?
There is indeed a native way: use AQ JMS from PL/SQL, see https://docs.oracle.com/database/121/ADQUE/jm_exmpl.htm#ADQUE1600. In short you create an AQ queue with a JMS payload type; then you can post messages with PL/SQL from the trigger. An external Java client can connect to the database and read the messages with JMS.
I don't know how much a call into Java would affect performance, but I try to avoid it. It was a nice idea but it never really caught on, so it remains a fringe case and at least early on there were always issues. PL/SQL on the other hand works.
If you need to send data to another message queue product (tags activemq and mq) you can read the messages in Java and forward them. It adds an extra step, but it is straightforward.
loadjava have many problems and not stable if there is many classes loaded and many business, take a look Calling Java from Oracle, PLSQL causing oracle.aurora.vm.ReadOnlyObjectException
Oracle AQ as i know is not free.
I have implemented the same need after trying many possibilities by creating only 1 class loaded to oracle with loadjava which is called as a procedure by a trigger and have the responsability to call an external java program with all needed parameters and log external process output to a table, as below.
i have encoded text mesage to BASE64 because i used JSON format and some specials caracters can causes problems as a parameters to external java program.
i have used "#*#jms_separator#*#" as a separator in the sent parameter string to parse the content if i need to send many parameters to the external program.
the whole duration of ShellExecutor.shellExec is around 500ms and running since 1 year without any problem.
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.sql.Clob;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.Arrays;
import java.util.concurrent.FutureTask;
import sun.misc.BASE64Encoder;
public class ShellExecutor {
static {
System.setProperty("file.encoding", "UTF-8");
}
private static final String INSERT_LOGS_SQL = "INSERT INTO JMS_LOG (TEXT_LOG) VALUES (?) ";
private static final String DEFAULT_CONNECTION = "jdbc:default:connection:";
public static String SQLshellExec(String command) throws Exception {
long start = System.currentTimeMillis();
StringBuffer result = new StringBuffer();
ShellExecutor worker = new ShellExecutor();
try {
worker.shellExec(command, result);
} finally {
result.append("exe duration : " + (System.currentTimeMillis() - start + "\n"));
Connection dbConnection = null;
PreparedStatement logsStatement = null;
try {
dbConnection = DriverManager.getConnection(DEFAULT_CONNECTION);
logsStatement = dbConnection.prepareStatement(INSERT_LOGS_SQL);
logsStatement.clearParameters();
Clob clob = dbConnection.createClob();
clob.setString(1, result.toString());
logsStatement.setClob(1, clob);
logsStatement.executeUpdate();
} finally {
if (logsStatement != null) {
try {
logsStatement.close();
} catch (Exception e) {
}
}
}
}
return result.substring(result.length() - 3090);
}
public void shellExec(String command, StringBuffer result) throws Exception {
Process process = null;
int exit = -10;
try {
InputStream stdout = null;
String[] params = command.split("#*#jms_separator#*#");
BASE64Encoder benc = new BASE64Encoder();
for (int i = 0; i < params.length; i++) {
if (params[i].contains("{") || params[i].contains("}") || params[i].contains("<")
|| params[i].contains("/>")) {
params[i] = benc.encodeBuffer(params[i].getBytes("UTF-8"));
}
}
result.append("Using separator : " + "#*#jms_separator#*#").append("\n")
.append("Calling : " + Arrays.toString(params)).append("\n");
ProcessBuilder pb = new ProcessBuilder(params);
pb.redirectErrorStream(true);
process = pb.start();
stdout = process.getInputStream();
LogStreamReader lsr = new LogStreamReader(stdout, result);
FutureTask<String> stdoutFuture = new FutureTask<String>(lsr, null);
Thread thread = new Thread(stdoutFuture, "LogStreamReader");
thread.start();
try {
exit = process.waitFor();
} catch (InterruptedException e) {
try {
exit = process.waitFor();
} catch (Exception e1) {
}
}
stdoutFuture.get();
result.append("\n").append("exit code :").append(exit).append("\n");
if (exit != 0) {
throw new RuntimeException(result.toString());
}
} catch (Exception e) {
result.append("\nException(").append(e.toString()).append("):").append(e.getCause()).append("\n\n");
e.printStackTrace(System.err);
throw e;
} finally {
if (process != null) {
process.destroy();
}
}
}
}
class LogStreamReader implements Runnable {
private BufferedReader reader;
private StringBuffer result;
public LogStreamReader(InputStream is, StringBuffer result) {
this.reader = new BufferedReader(new InputStreamReader(is));
this.result = result;
}
public void run() {
try {
String line = null;
while ((line = reader.readLine()) != null) {
result.append(line).append("\n");
}
} catch (Exception e) {
result.append("\nException(").append(e.toString()).append("):").append(e.getCause()).append("\n\n");
e.printStackTrace(System.err);
} finally {
try {
reader.close();
} catch (IOException e) {
}
}
}
}
The class of the external Java program packaged as an executable with all needed librairies, a simple JMS sender :
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import javax.jms.Destination;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.apache.commons.codec.binary.Base64;
import org.json.JSONObject;
import progress.message.jclient.ConnectionFactory;
import progress.message.jimpl.Connection;
public class JMSSender {
private static SimpleDateFormat sdf = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss.SSS");
public static void main(String[] args) throws Throwable {
doSend(args[0]);
}
public static void doSend(String text)
throws Throwable {
if (Base64.isBase64(text)) {
text = new String(Base64.decodeBase64(text));
}
String content = "\n\nsending message :" + text;
Connection con = null;
Session session = null;
try {
ConnectionFactory cf = new ConnectionFactory();
session = con.createSession(false, Session.AUTO_ACKNOWLEDGE);
Destination dest = session.createTopic(destination) ;
MessageProducer producer = session.createProducer(dest);
con.start();
JSONObject json = new JSONObject();
json.put("content", text);
json.put("date", sdf.format(new Date()));
TextMessage tm = session.createTextMessage(json.toString());
producer.send(tm);
content += " \n\n" + "sent message :" + json.toString();
} catch (Throwable e) {
content += " \n\n" + e.toString() + " \n\n" + Arrays.toString(e.getStackTrace());
if (e.getCause() != null) {
content += " \n\nCause : " + e.getCause().toString() + " \n\n"
+ Arrays.toString(e.getCause().getStackTrace());
}
e.printStackTrace(System.err);
throw e;
} finally {
write("steps on sending message : " + content);
if (session != null) {
try {
session.commit();
session.close();
} catch (Exception e) {
}
session = null;
}
if (con != null) {
try {
con.stop();
con.close();
} catch (Exception e) {
}
}
}
}
private static void write(String log) {
try {
if (System.out != null) {
System.out.println(log);
}
} catch (Exception e2) {
}
}
}

when i start .jar app on mac do nothing

I have made a simple JavaFx application, wich read/write the txt file (the code is below). I have tested it on windows platform (7 and higher).
After i gave app my friend for test on Mac. But when we run it, just do nothing. The GUI did not appear.
I tried run it through terminal (just "drug and drop" the icon to terminal, then enter in teminal. I don't know did i do right?) and terminal returned message "Permission denied". Can somebody explain what is require to run application, and what is not permitted exactly?
I found the similar question there but it have not answer...
Code:
import javafx.application.*;
import javafx.concurrent.Task;
import javafx.geometry.Orientation;
import javafx.geometry.Pos;
import javafx.scene.*;
import javafx.scene.control.Label;
import javafx.scene.control.SplitPane;
import javafx.scene.text.Font;
import javafx.stage.*;
import javafx.scene.layout.*;
import java.io.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
public class Main extends Application {
static Properties props;
static double fontSize;
static String charsetName;
static long mills;
static String delimiter;
static String pathToDictionary;
static String pathErrLog;
static {
String pathFileProps = System.getProperty("user.dir") + "\\props.txt";
props = new Properties();
try {
FileInputStream fis = new FileInputStream(pathFileProps);
props.load(fis);
fontSize = Double.parseDouble(props.getProperty("Font"));
charsetName = props.getProperty("Charset");
mills = Long.parseLong(props.getProperty("mills"));
delimiter = props.getProperty("delimiter");
pathToDictionary = props.getProperty("dict.path");
fis.close();
} catch (IOException e) {
try {
props.put("Font", "20");
props.put("Charset", "UTF-8");
props.put("mills", "1000");
props.put("delimiter", "\t");
props.put("dict.path", System.getProperty("user.dir") + "\\dict.txt");
System.out.println("dictPath:" + System.getProperty("user.dir") + "\\dict.txt");
System.setProperty("file.encoding", "UTF-8");
FileOutputStream fos = new FileOutputStream(pathFileProps);
props.store(fos
, "Props description:" + "\n" +
"Font" + "\t\t" + "size of font's words " + "\n" +
"Charset" + "\t" + "charset of file-dictionary" + "\n" +
"mills" + "\t\t" + "per animations in milliseconds" + "\n" +
"delimiter" + "\t" + "delimiter between a pair words in string: eng<->rus \"<->\" is delimiter there." + "\n" +
"dict.path" + "\t" + "path to file-dictionary. Use \"/\"-symbol in path. Ex: C:/temp/dict.txt" + "\n" +
"\t\t\t" + "Use only eng symbols to set path!" + "\n" +
"\tYou can change only values!\n");
fos.close();
System.exit(0);
} catch (IOException e1) {
errPrint(e1,"Ошибка создания файла: " + pathFileProps + "\n");
}
}
}
ArrayList<String> dictionary = new ArrayList<>();
public static void errPrint(Exception exc, String note) {
if (pathErrLog == null) {
pathErrLog = System.getProperty("user.dir") + "\\errors.log";
}
try {
PrintWriter outFile = new PrintWriter(new FileWriter(pathErrLog, true));
outFile.println(note);
exc.printStackTrace(outFile);
outFile.close();
System.exit(0);
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
launch(args);
}
public void init() throws IOException {
try {
BufferedReader reader = new BufferedReader(new FileReader(pathToDictionary));
int cnt = 0;
while (reader.ready()) {
String line = new String(reader.readLine().getBytes(), Charset.forName(charsetName));
dictionary.add(line);
}
} catch (FileNotFoundException e) {
errPrint(e,"Не найден файл " + pathToDictionary);
} catch (IOException e) {
errPrint(e,"Ошибка чтения файла " + pathToDictionary);
}
}
public void start(Stage myStage) {
myStage.setTitle("WordsLearner");
SplitPane sp = new SplitPane();
sp.setOrientation(Orientation.VERTICAL);
Label labelUp = new Label();
Label labelDw = new Label();
labelUp.setFont(new Font(fontSize));
labelDw.setFont(new Font(fontSize));
Scene scene = new Scene(sp, 600, 200);
myStage.setScene(scene);
final StackPane sp1 = new StackPane();
sp1.getChildren().add(labelUp);
sp1.setAlignment(Pos.BOTTOM_CENTER);
final StackPane sp2 = new StackPane();
sp2.getChildren().add(labelDw);
sp2.setAlignment(Pos.TOP_CENTER);
final boolean[] flag = {true};
sp.setOnMouseClicked(event -> {
Task<Void> task = new Task<Void>() {
#Override
public Void call() throws Exception {
if (flag[0]) {
flag[0] = false;
final String[] str = new String[1];
final String[] eng = new String[1];
final String[] rus = new String[1];
while (true) {
Platform.runLater(() -> {
try {
str[0] = dictionary.get(ThreadLocalRandom.current().nextInt(0, dictionary.size()));
eng[0] = str[0].split(delimiter)[0];
rus[0] = str[0].split(delimiter)[1];
labelUp.setText(eng[0]);
labelDw.setText(rus[0]);
} catch (Exception e) {
System.exit(-1);
}
});
Thread.sleep(mills);
}
}
return null;
}
};
new Thread(task).start();
});
sp.getItems().addAll(sp1, sp2);
sp.setDividerPositions(0.5f, 0.5f);
myStage.show();
}
public void stop() {
System.exit(0);
}
}

Check for broken links using java code and link is not getting redirect

This code will check my link is broken or not and validateWebResourceUrl method will return 0 status .even if i get 200 status my link couldn't redirect to mozilla firefox when i call my api call.Below code will show u my api cal.
#RequestMapping(method = RequestMethod.POST, value = "/validate/fburl")
public ModelAndView validateFbLink(HttpServletRequest request,#RequestParam(value="key", required=true) String key ,#RequestParam(value = "link") String link, HttpServletResponse response) throws Exception {
ModelAndView jsonmodel = new ModelAndView("rest/model");
jsonmodel.addObject("model", httpFrameBreakChecker.validateHttpLink(link, key));
return jsonmodel;
}
This api call will go to check the link is broken or not ,if not it will go to redirect.I am trying to find the broken links and i am getting status 200 for that link but that link is not redirecting.Below is my code.
package org.ednovo.gooru.util;
import org.springframework.stereotype.Component;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebResponse;
#Component
public class HttpFrameBreakChecker extends LinkChecker {
#Override
public Integer validateHttpLink(String resourceUrl, String key) {
Integer status = null;
status = validateWebResourceUrl(resourceUrl, getPageTesterPath());
putResourceStatus(status, key);
return status;
}
public synchronized Integer validateWebResourceUrl(String webURL, String pageTesterURL) {
WebClient webClient = null;
WebResponse webResponse = null;
try {
webClient = new WebClient(BrowserVersion.FIREFOX_3_6);
webClient.setCssEnabled(false);
webClient.setThrowExceptionOnScriptError(false);
webResponse = webClient.getPage(webURL).getWebResponse();
int i = webResponse.getStatusCode();
logger.info(i+"");
if (webResponse.getStatusCode() != 200) {
return null;
}
String urlToCheck = webURL + pageTesterURL;
// Now check if the page has a frame breaker.
webResponse = webClient.getPage(urlToCheck).getWebResponse();
int status = webResponse.getStatusCode();
logger.info(status+""+"frame");
String pageURL = webResponse.getWebRequest().getUrl().toString();
int validationStatus = (urlToCheck.equals(pageURL) && status == 200) ? 0 : 1;
logger.info(validationStatus+ " return statement of validateWebResourceUrl ");
return validationStatus;
} catch (Exception exception) {
logger.error("ERROR : Unable to start web driver : " + exception.getMessage());
return null;
} finally {
try {
if (webClient != null) {
webClient.closeAllWindows();
}
} catch (Exception ex) {
}
}
}
#Override
public void commitPage() {
if (resources.size() > 0) {
String resourcesString = getxStream().toXML(resources);
postData(getApiPath(), "/resource/urls/update/frameBreaker/" + getGlobalJobKey(), resourcesString);
postData(getSearchApiPath(), "/search/resource/update/index/" + getGlobalJobKey(), resourcesString );
}
}
#Override
public String getType() {
return "FB";
}
}
And Below code will show you the LinkChecker class.
package org.ednovo.gooru.util;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.restlet.data.Form;
import org.restlet.representation.Representation;
import org.restlet.resource.ClientResource;
import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.io.xml.DomDriver;
public abstract class LinkChecker extends BaseComponent {
private int pageSize = 100;
private Map<String, String> configSettings;
private XStream xStream = new XStream(new DomDriver());
protected Map<Integer, List<String>> resources = new HashMap<Integer, List<String>>();
private String listResponse = null;
private Map<Long, String> resourceUrls = null;
public final synchronized void execute() {
reset();
ClientResource configResource = null;
Representation representation = null;
try {
configResource = createClientResource(getApiPath() + "/config-settings");
representation = configResource.get();
String settingsText = representation.getText();
configSettings = (Map<String, String>) xStream.fromXML(settingsText);
if (isLinkCheckerSchedulerEnabled()) {
run();
}
} catch (Exception exception) {
logger.error(exception.getMessage());
} finally {
releaseClientResources(configResource, representation);
}
}
public void run() throws Exception {
reset();
int page = 0;
while (true) {
++page;
listResponse = null;
resourceUrls = null;
final int currentPage = page;
ClientResourceExecuter clientResourceExecuter = new ClientResourceExecuter() {
#Override
public void run(ClientResource clientResource, Representation representation) throws Exception {
reset();
clientResource = createClientResource(getSearchApiPath() +"/search/resource-url-check?category="+getType()+"&pageSize="+pageSize+"&format=xml");
representation = clientResource.get();
listResponse = representation.getText();
if (listResponse != null || listResponse.length() > 1) {
try {
resourceUrls = (Map<Long, String>) xStream.fromXML(listResponse);
} catch (Exception ex) {
logger.error("Conversion of json to list failed in " + getType() + " validation");
}
logger.warn("Validating " + getType() + " Urls of Page : " + currentPage + " with size : " + (resourceUrls != null ? resourceUrls.size() : 0));
validateResourceUrls(resourceUrls);
}
}
};
clientResourceExecuter = null;
listResponse = null;
if (resourceUrls == null || resourceUrls.size() == 0) {
break;
}
if (getType().equals("HTTP") && page >= 5) {
break;
}else if (getType().equals("FB") && page >= 1) {
break;
}
}
}
public Integer validateHttpLink(String link, String key) {
URL url = null;
URLConnection connection = null;
HttpURLConnection httpConnection = null;
Integer responseCode = null;
try {
url = new URL(link);
connection = url.openConnection();
connection.connect();
httpConnection = (HttpURLConnection) connection;
httpConnection.setConnectTimeout(60000);
responseCode = httpConnection.getResponseCode();
if (responseCode != 200) {
logger.error("Url Checker validation fails for resource : " + link + " : " + responseCode);
}
} catch (MalformedURLException e) {
logger.error("Error in link checker : " + link + " Message : " + e.getMessage());
return 404;
} catch (IOException e) {
logger.error("Error in link checker : " + link + " Message : " + e.getMessage());
return 404;
} catch (Exception ex) {
logger.error("Error in link checker : " + link + " Message : " + ex.getMessage());
return null;
} finally {
try {
if (httpConnection != null) {
httpConnection.disconnect();
}
} catch (Exception ex) {
}
}
return responseCode;
}
protected void postData(String path, String url, String data) {
Form form = new Form();
form.add("data", data);
createAndRunClientResource(path, url, form);
}
public void validateResourceUrls(Map<Long, String> resourceUrls) throws Exception {
if (resourceUrls.size() > 0) {
Iterator<?> iterator = resourceUrls.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<Long, String> entry = (Map.Entry<Long, String>) iterator.next();
String resourceUrl = entry.getValue();
if (getType().equals("FB")) {
Thread.sleep(1000);
} else {
Thread.sleep(900);
}
logger.debug("Validating " + getType() + " url - " + resourceUrl);
Integer status = validateHttpLink(resourceUrl, entry.getKey()+"");
putResourceStatus(status, entry.getKey()+"");
}
commitPage();
}
}
public void commitPage() throws Exception {
if (resources.size() > 0) {
String resourcesString = getxStream().toXML(resources);
postData(getApiPath(), "/resource/urls/checker/update/" + getGlobalJobKey(), resourcesString);
//postData(getSearchApiPath(), "/search/resource/update/index/" + getGlobalJobKey(), resourcesString);
}
}
public void putResourceStatus(Integer status, String resourceId) {
if (status == null) {
status = -99;
}
List<String> resourceIds = null;
if (resources.containsKey(status)) {
resourceIds = resources.get(status);
} else {
resourceIds = new ArrayList<String>();
resources.put(status, resourceIds);
}
resourceIds.add(resourceId);
}
public void reset() {
resources.clear();
}
public boolean isLinkCheckerSchedulerEnabled() {
if (configSettings != null && configSettings.containsKey("urlChecker.scheduler.enabled")) {
return configSettings.get("urlChecker.scheduler.enabled").equals("true") ? true : false;
} else {
return false;
}
}
public abstract String getType();
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public XStream getxStream() {
return xStream;
}
public static void main(String args[]) {
}
}

Resources