Check for broken links using java code and link is not getting redirect - htmlunit

Question

This code will check my link is broken or not and validateWebResourceUrl method will return 0 status .even if i get 200 status my link couldn't redirect to mozilla firefox when i call my api call.Below code will show u my api cal.
#RequestMapping(method = RequestMethod.POST, value = "/validate/fburl")
public ModelAndView validateFbLink(HttpServletRequest request,#RequestParam(value="key", required=true) String key ,#RequestParam(value = "link") String link, HttpServletResponse response) throws Exception {
ModelAndView jsonmodel = new ModelAndView("rest/model");
jsonmodel.addObject("model", httpFrameBreakChecker.validateHttpLink(link, key));
return jsonmodel;
}
This api call will go to check the link is broken or not ,if not it will go to redirect.I am trying to find the broken links and i am getting status 200 for that link but that link is not redirecting.Below is my code.
package org.ednovo.gooru.util;
import org.springframework.stereotype.Component;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebResponse;
#Component
public class HttpFrameBreakChecker extends LinkChecker {
#Override
public Integer validateHttpLink(String resourceUrl, String key) {
Integer status = null;
status = validateWebResourceUrl(resourceUrl, getPageTesterPath());
putResourceStatus(status, key);
return status;
}
public synchronized Integer validateWebResourceUrl(String webURL, String pageTesterURL) {
WebClient webClient = null;
WebResponse webResponse = null;
try {
webClient = new WebClient(BrowserVersion.FIREFOX_3_6);
webClient.setCssEnabled(false);
webClient.setThrowExceptionOnScriptError(false);
webResponse = webClient.getPage(webURL).getWebResponse();
int i = webResponse.getStatusCode();
logger.info(i+"");
if (webResponse.getStatusCode() != 200) {
return null;
}
String urlToCheck = webURL + pageTesterURL;
// Now check if the page has a frame breaker.
webResponse = webClient.getPage(urlToCheck).getWebResponse();
int status = webResponse.getStatusCode();
logger.info(status+""+"frame");
String pageURL = webResponse.getWebRequest().getUrl().toString();
int validationStatus = (urlToCheck.equals(pageURL) && status == 200) ? 0 : 1;
logger.info(validationStatus+ " return statement of validateWebResourceUrl ");
return validationStatus;
} catch (Exception exception) {
logger.error("ERROR : Unable to start web driver : " + exception.getMessage());
return null;
} finally {
try {
if (webClient != null) {
webClient.closeAllWindows();
}
} catch (Exception ex) {
}
}
}
#Override
public void commitPage() {
if (resources.size() > 0) {
String resourcesString = getxStream().toXML(resources);
postData(getApiPath(), "/resource/urls/update/frameBreaker/" + getGlobalJobKey(), resourcesString);
postData(getSearchApiPath(), "/search/resource/update/index/" + getGlobalJobKey(), resourcesString );
}
}
#Override
public String getType() {
return "FB";
}
}
And Below code will show you the LinkChecker class.
package org.ednovo.gooru.util;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.restlet.data.Form;
import org.restlet.representation.Representation;
import org.restlet.resource.ClientResource;
import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.io.xml.DomDriver;
public abstract class LinkChecker extends BaseComponent {
private int pageSize = 100;
private Map<String, String> configSettings;
private XStream xStream = new XStream(new DomDriver());
protected Map<Integer, List<String>> resources = new HashMap<Integer, List<String>>();
private String listResponse = null;
private Map<Long, String> resourceUrls = null;
public final synchronized void execute() {
reset();
ClientResource configResource = null;
Representation representation = null;
try {
configResource = createClientResource(getApiPath() + "/config-settings");
representation = configResource.get();
String settingsText = representation.getText();
configSettings = (Map<String, String>) xStream.fromXML(settingsText);
if (isLinkCheckerSchedulerEnabled()) {
run();
}
} catch (Exception exception) {
logger.error(exception.getMessage());
} finally {
releaseClientResources(configResource, representation);
}
}
public void run() throws Exception {
reset();
int page = 0;
while (true) {
++page;
listResponse = null;
resourceUrls = null;
final int currentPage = page;
ClientResourceExecuter clientResourceExecuter = new ClientResourceExecuter() {
#Override
public void run(ClientResource clientResource, Representation representation) throws Exception {
reset();
clientResource = createClientResource(getSearchApiPath() +"/search/resource-url-check?category="+getType()+"&pageSize="+pageSize+"&format=xml");
representation = clientResource.get();
listResponse = representation.getText();
if (listResponse != null || listResponse.length() > 1) {
try {
resourceUrls = (Map<Long, String>) xStream.fromXML(listResponse);
} catch (Exception ex) {
logger.error("Conversion of json to list failed in " + getType() + " validation");
}
logger.warn("Validating " + getType() + " Urls of Page : " + currentPage + " with size : " + (resourceUrls != null ? resourceUrls.size() : 0));
validateResourceUrls(resourceUrls);
}
}
};
clientResourceExecuter = null;
listResponse = null;
if (resourceUrls == null || resourceUrls.size() == 0) {
break;
}
if (getType().equals("HTTP") && page >= 5) {
break;
}else if (getType().equals("FB") && page >= 1) {
break;
}
}
}
public Integer validateHttpLink(String link, String key) {
URL url = null;
URLConnection connection = null;
HttpURLConnection httpConnection = null;
Integer responseCode = null;
try {
url = new URL(link);
connection = url.openConnection();
connection.connect();
httpConnection = (HttpURLConnection) connection;
httpConnection.setConnectTimeout(60000);
responseCode = httpConnection.getResponseCode();
if (responseCode != 200) {
logger.error("Url Checker validation fails for resource : " + link + " : " + responseCode);
}
} catch (MalformedURLException e) {
logger.error("Error in link checker : " + link + " Message : " + e.getMessage());
return 404;
} catch (IOException e) {
logger.error("Error in link checker : " + link + " Message : " + e.getMessage());
return 404;
} catch (Exception ex) {
logger.error("Error in link checker : " + link + " Message : " + ex.getMessage());
return null;
} finally {
try {
if (httpConnection != null) {
httpConnection.disconnect();
}
} catch (Exception ex) {
}
}
return responseCode;
}
protected void postData(String path, String url, String data) {
Form form = new Form();
form.add("data", data);
createAndRunClientResource(path, url, form);
}
public void validateResourceUrls(Map<Long, String> resourceUrls) throws Exception {
if (resourceUrls.size() > 0) {
Iterator<?> iterator = resourceUrls.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<Long, String> entry = (Map.Entry<Long, String>) iterator.next();
String resourceUrl = entry.getValue();
if (getType().equals("FB")) {
Thread.sleep(1000);
} else {
Thread.sleep(900);
}
logger.debug("Validating " + getType() + " url - " + resourceUrl);
Integer status = validateHttpLink(resourceUrl, entry.getKey()+"");
putResourceStatus(status, entry.getKey()+"");
}
commitPage();
}
}
public void commitPage() throws Exception {
if (resources.size() > 0) {
String resourcesString = getxStream().toXML(resources);
postData(getApiPath(), "/resource/urls/checker/update/" + getGlobalJobKey(), resourcesString);
//postData(getSearchApiPath(), "/search/resource/update/index/" + getGlobalJobKey(), resourcesString);
}
}
public void putResourceStatus(Integer status, String resourceId) {
if (status == null) {
status = -99;
}
List<String> resourceIds = null;
if (resources.containsKey(status)) {
resourceIds = resources.get(status);
} else {
resourceIds = new ArrayList<String>();
resources.put(status, resourceIds);
}
resourceIds.add(resourceId);
}
public void reset() {
resources.clear();
}
public boolean isLinkCheckerSchedulerEnabled() {
if (configSettings != null && configSettings.containsKey("urlChecker.scheduler.enabled")) {
return configSettings.get("urlChecker.scheduler.enabled").equals("true") ? true : false;
} else {
return false;
}
}
public abstract String getType();
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public XStream getxStream() {
return xStream;
}
public static void main(String args[]) {
}
}