everything was working flawlessly like 3 days ago when I finished writing some code and testing. Today when I wanted to do some more changes it stopped working.. Like without a reason. Code didn't change a bit. I'm getting this error:
Uncaught RangeError: Maximum call stack size exceeded
Debugger tells me that error has been thrown somewhere during this call:
return open.apply(this, arguments);
Because of that anything that should be loaded via AJAX on the site is not working.
This is the code for intercepting the AJAX calls:
var XHR = XMLHttpRequest.prototype;
var open = XHR.open;
var send = XHR.send;
var setRequestHeader = XHR.setRequestHeader;
XHR.open = function (method, url) {
this._url = url;
this._requestHeaders = {};
return open.apply(this, arguments);
};
XHR.setRequestHeader = function (header, value) {
this._requestHeaders[header] = value;
return setRequestHeader.apply(this, arguments);
};
XHR.send = function (postData) {
this.addEventListener('load', function () {
var myUrl = this._url ? this._url.toLowerCase() : this._url;
if (myUrl) {
if (this.responseType != 'blob' && this.responseText) {
try {
var allowedUrls = ['shoutbox.json', 'comments/latest.json'];
if (new RegExp(allowedUrls.join("|")).test(this._url)) {
var extensionID = 'exampleExtensionID';
chrome.runtime.sendMessage(extensionID, { interception: true });
}
} catch (err) {
}
}
}
});
return send.apply(this, arguments);
};
Has anyone idea why it might stopped working?
I had to put my injected.js (XHR) into self invoking function.
I am facing problem with my code in FireFox and Safari as below:
xhr = new window['XMLHttpRequest'];
xhr.onreadystatechange = function() {
if (done || xhr.readyState != 4) {
return;
}
done = true;
handleResponse(xhr.responseText, callback);
};
}
xhr.open('GET', uri+params, true);
xhr.withCredentials = true;
xhr.send(null);
function handleResponse(responseText, callback) {
var error;
var result;
try {
result = toucan.JSON.parse(responseText)['result']; //connectedAuth
logout result.
} catch (ex) {
result = undefined;
}
console.log("Result is" + result);
if (!result) {
var errorCode = 'UnknownError';
var errorMessage = 'An unknown error ocurred';
error = toucan.Base.format('%s: %s', errorCode, errorMessage);
}
invokeCallback(error, callback);
}
This is followed by redirection as :window.location.href = "index.php?module=login&method=logout";
However, I am not getting any response back from the request I made if it is followed by redirection in FireFox.
This works fine in Chrome but not in Firefox and is specific to the case when request is followed by redirection.
I do not have control on redirection code to be changed. Is there a way that the browser can be enforced to first complete the request and get the response before going for redirection while keeping the call asynchronous.
I would suggest you to use a promise, first create a function that run the ajax call that return the response from your server:
ajax_AuthUser(id,pass){
return $.ajax({
method: "POST",
url: "authUser.php",
data: { id: id, pass: pass}
})
}
Second use a done statement:
ajax_AuthUser(id,pass)
.done(function(response){
//check the response here !! maybe validate the json ?
var auth = JSON.parse(response)
if(auth.response == "approved"){
//do something here
}else{
//do other stuff here
}
}).fail(function(response){
//do something if fail
}).always(function(){
//do something after the call finished
})
If you want a live example here is a jsfiddle that show how promises work
Hope it helps
The data on the webpage is displayed dynamically and it seems that checking for every change in the html and extracting the data is a very daunting task and also needs me to use very unreliable XPaths. So I would want to be able to extract the data from the XHR packets.
I hope to be able to extract information from XHR packets as well as generate 'XHR' packets to be sent to the server.
The extracting information part is more important for me because the sending of information can be handled easily by automatically triggering html elements using casperjs.
I'm attaching a screenshot of what I mean.
The text in the response tab is the data I need to process afterwards. (This XHR response has been received from the server.)
This is not easily possible, because the resource.received event handler only provides meta data like url, headers or status, but not the actual data. The underlying phantomjs event handler acts the same way.
Stateless AJAX Request
If the ajax call is stateless, you may repeat the request
casper.on("resource.received", function(resource){
// somehow identify this request, here: if it contains ".json"
// it also also only does something when the stage is "end" otherwise this would be executed two times
if (resource.url.indexOf(".json") != -1 && resource.stage == "end") {
var data = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, resource.url);
// do something with data, you might need to JSON.parse(data)
}
});
casper.start(url); // your script
You may want to add the event listener to resource.requested. That way you don't need to way for the call to complete.
You can also do this right inside of the control flow like this (source: A: CasperJS waitForResource: how to get the resource i've waited for):
casper.start(url);
var res, resData;
casper.waitForResource(function check(resource){
res = resource;
return resource.url.indexOf(".json") != -1;
}, function then(){
resData = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, res.url);
// do something with the data here or in a later step
});
casper.run();
Stateful AJAX Request
If it is not stateless, you would need to replace the implementation of XMLHttpRequest. You will need to inject your own implementation of the onreadystatechange handler, collect the information in the page window object and later collect it in another evaluate call.
You may want to look at the XHR faker in sinon.js or use the following complete proxy for XMLHttpRequest (I modeled it after method 3 from How can I create a XMLHttpRequest wrapper/proxy?):
function replaceXHR(){
(function(window, debug){
function args(a){
var s = "";
for(var i = 0; i < a.length; i++) {
s += "\t\n[" + i + "] => " + a[i];
}
return s;
}
var _XMLHttpRequest = window.XMLHttpRequest;
window.XMLHttpRequest = function() {
this.xhr = new _XMLHttpRequest();
}
// proxy ALL methods/properties
var methods = [
"open",
"abort",
"setRequestHeader",
"send",
"addEventListener",
"removeEventListener",
"getResponseHeader",
"getAllResponseHeaders",
"dispatchEvent",
"overrideMimeType"
];
methods.forEach(function(method){
window.XMLHttpRequest.prototype[method] = function() {
if (debug) console.log("ARGUMENTS", method, args(arguments));
if (method == "open") {
this._url = arguments[1];
}
return this.xhr[method].apply(this.xhr, arguments);
}
});
// proxy change event handler
Object.defineProperty(window.XMLHttpRequest.prototype, "onreadystatechange", {
get: function(){
// this will probably never called
return this.xhr.onreadystatechange;
},
set: function(onreadystatechange){
var that = this.xhr;
var realThis = this;
that.onreadystatechange = function(){
// request is fully loaded
if (that.readyState == 4) {
if (debug) console.log("RESPONSE RECEIVED:", typeof that.responseText == "string" ? that.responseText.length : "none");
// there is a response and filter execution based on url
if (that.responseText && realThis._url.indexOf("whatever") != -1) {
window.myAwesomeResponse = that.responseText;
}
}
onreadystatechange.call(that);
};
}
});
var otherscalars = [
"onabort",
"onerror",
"onload",
"onloadstart",
"onloadend",
"onprogress",
"readyState",
"responseText",
"responseType",
"responseXML",
"status",
"statusText",
"upload",
"withCredentials",
"DONE",
"UNSENT",
"HEADERS_RECEIVED",
"LOADING",
"OPENED"
];
otherscalars.forEach(function(scalar){
Object.defineProperty(window.XMLHttpRequest.prototype, scalar, {
get: function(){
return this.xhr[scalar];
},
set: function(obj){
this.xhr[scalar] = obj;
}
});
});
})(window, false);
}
If you want to capture the AJAX calls from the very beginning, you need to add this to one of the first event handlers
casper.on("page.initialized", function(resource){
this.evaluate(replaceXHR);
});
or evaluate(replaceXHR) when you need it.
The control flow would look like this:
function replaceXHR(){ /* from above*/ }
casper.start(yourUrl, function(){
this.evaluate(replaceXHR);
});
function getAwesomeResponse(){
return this.evaluate(function(){
return window.myAwesomeResponse;
});
}
// stops waiting if window.myAwesomeResponse is something that evaluates to true
casper.waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
});
casper.run();
As described above, I create a proxy for XMLHttpRequest so that every time it is used on the page, I can do something with it. The page that you scrape uses the xhr.onreadystatechange callback to receive data. The proxying is done by defining a specific setter function which writes the received data to window.myAwesomeResponse in the page context. The only thing you need to do is retrieving this text.
JSONP Request
Writing a proxy for JSONP is even easier, if you know the prefix (the function to call with the loaded JSON e.g. insert({"data":["Some", "JSON", "here"],"id":"asdasda")). You can overwrite insert in the page context
after the page is loaded
casper.start(url).then(function(){
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
or before the request is received (if the function is registered just before the request is invoked)
casper.on("resource.requested", function(resource){
// filter on the correct call
if (resource.url.indexOf(".jsonp") != -1) {
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}
}).run();
casper.start(url).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
I may be late into the party, but the answer may help someone like me who would fall into this problem later in future.
I had to start with PhantomJS, then moved to CasperJS but finally settled with SlimerJS. Slimer is based on Phantom, is compatible with Casper, and can send you back the response body using the same onResponseReceived method, in "response.body" part.
Reference: https://docs.slimerjs.org/current/api/webpage.html#webpage-onresourcereceived
#Artjom's answer's doesn't work for me in the recent Chrome and CasperJS versions.
Based on #Artjom's answer and based on gilly3's answer on how to replace XMLHttpRequest, I have composed a new solution that should work in most/all versions of the different browsers. Works for me.
SlimerJS cannot work on newer version of FireFox, therefore no good for me.
Here is the the generic code to add a listner to load of XHR (not dependent on CasperJS):
var addXHRListener = function (XHROnStateChange) {
var XHROnLoad = function () {
if (this.readyState == 4) {
XHROnStateChange(this)
}
}
var open_original = XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open = function (method, url, async, unk1, unk2) {
this.requestUrl = url
open_original.apply(this, arguments);
};
var xhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function () {
var xhr = this;
if (xhr.addEventListener) {
xhr.removeEventListener("readystatechange", XHROnLoad);
xhr.addEventListener("readystatechange", XHROnLoad, false);
} else {
function readyStateChange() {
if (handler) {
if (handler.handleEvent) {
handler.handleEvent.apply(xhr, arguments);
} else {
handler.apply(xhr, arguments);
}
}
XHROnLoad.apply(xhr, arguments);
setReadyStateChange();
}
function setReadyStateChange() {
setTimeout(function () {
if (xhr.onreadystatechange != readyStateChange) {
handler = xhr.onreadystatechange;
xhr.onreadystatechange = readyStateChange;
}
}, 1);
}
var handler;
setReadyStateChange();
}
xhrSend.apply(xhr, arguments);
};
}
Here is CasperJS code to emit a custom event on load of XHR:
casper.on("page.initialized", function (resource) {
var emitXHRLoad = function (xhr) {
window.callPhantom({eventName: 'xhr.load', eventData: xhr})
}
this.evaluate(addXHRListener, emitXHRLoad);
});
casper.on('remote.callback', function (data) {
casper.emit(data.eventName, data.eventData)
});
Here is a code to listen to "xhr.load" event and get the XHR response body:
casper.on('xhr.load', function (xhr) {
console.log('xhr load', xhr.requestUrl)
console.log('xhr load', xhr.responseText)
});
Additionally, you can also directly download the content and manipulate it later.
Here is the example of the script I am using to retrieve a JSON and save it locally :
var casper = require('casper').create({
pageSettings: {
webSecurityEnabled: false
}
});
var url = 'https://twitter.com/users/username_available?username=whatever';
casper.start('about:blank', function() {
this.download(url, "hop.json");
});
casper.run(function() {
this.echo('Done.').exit();
});
I've seen the myriad threads sprawled across the Internet about the following similar code in an AJAX request returning undefined:
AJAX.onreadystatechange = function() {
if(AJAX.readyState == 4) {
if(AJAX.status == 200) {
var response = AJAX.responseText;
return response;
}
else {
window.alert('Error: ' + AJAX.status);
return false;
}
}
};
I know that I'm supposed to "do something with" responseText like writing it to the HTML. The problem: I don't have that luxury. This bit of code is intended to be inside of a generic method for running fast AJAX requests that way all the code for making an AJAX request doesn't have to written out over and over again (~40×) with the chance of a minor problem here or there that breaks the application.
My method HAS to explicitly return responseText "or else." No writing to HTML. How would I do this? Also, I'd appreciate a lack of plugs for JQuery.
What I'm looking for:
function doAjax(param) {
// set up XmlHttpRequest
AJAX.onreadystatechange = function() {
if(AJAX.readyState == 4) {
if(AJAX.status == 200) {
var response = AJAX.responseText;
return response;
}
else {
window.alert('Error: ' + AJAX.status);
return false;
}
}
};
// send data
}
...
function doSpecificAjax() {
var param = array();
var result = doAjax(param);
// manipulate result
}
Doing a little research I came across this SOF post:
Ajax responseText comes back as undefined
Based on that post, it looks like you may want to implement your ajax method like this:
function doRequest(url, callback) {
var xmlhttp = ....; // create a new request here
xmlhttp.open("GET", url, true); // for async
xmlhttp.onreadystatechange=function() {
if (xmlhttp.readyState==4) {
if (xmlhttp.status == 200) {
// pass the response to the callback function
callback(null, xmlhttp.responseText);
} else {
// pass the error to the callback function
callback(xmlhttp.statusText);
}
}
}
xmlhttp.send(null);
}
Then, you can call that method like this...
doRequest('http://mysite.com/foo', function(err, response) { // pass an anonymous function
if (err) {
return "";
} else {
return response;
}
});
This should return the responseText accurately. Let me know if this doesn't give you back the correct results.
I've been trying to make this mootools (ver: 1.2.4) Class to process my ajax requests. But my scripts only returns readystate 1. never gets to 2,3 ,or 4, the method handleHttpResponse only seems to run once. I'ves put alerts to see and I only get 1. any ideas?
var replyXML = new Class ({
/* GDW AJAX REQUEST SCRIPT */
/* By: Jonathan Robidas 2011-05-13 */
initialize : function(url){
this.http = this.getHTTPObject(); // We create the HTTP Object
this.url = url; // Requested server-side script
this.response = ''; // String returned by AJAX, Set after positive response
},
returnResponse : function() {
return this.response;
},
handleHttpResponse : function() {
alert(this.http.readyState);
if (this.http.readyState == 4) {
if(this.http.status==200) {
alert("YA YA 2");
this.response = this.http.responseText;
return true;
}
}
},
requestXML : function() {
this.http.open("GET", this.url, true);
//this.http.onreadystateshange = this.handleHttpResponse();
this.http.onload = this.handleHttpResponse();
},
getHTTPObject : function() {
var xmlhttp;
if(window.XMLHttpRequest){
xmlhttp = new XMLHttpRequest();
}
else if (window.ActiveXObject){
xmlhttp=new ActiveXObject("Microsoft.XMLHTTP");
if (!xmlhttp){
xmlhttp=new ActiveXObject("Msxml2.XMLHTTP");
}
}
return xmlhttp;
}
});
This is how i launch it for now. my content is null but my url is a working XML file. so it shouldnt be blank... ?
<script language="Javascript" type="text/Javascript">
window.addEvent('domready', function() {
loadXML = new replyXML('gdwcommreply_genxml.php?getfield=idvideo&fieldid=64&parentid=59');
loadXML.requestXML();
content = loadXML.returnResponse();
alert(content);
/*
x=content.documentElement.childNodes;
for (i=0;i<x.length;i++) {
document.write(x[i].nodeName);
document.write(": ");
document.write(x[i].childNodes[0].nodeValue);
document.write("<br />");
}
*/
});
</script>
Tested on Google chrome, Firefox 4, Internet explorer 7 & 8, all same result.
Here is an example of XML the script is suppose to output: http://jerenovici.net/gdwcommreply_genxml.php?getfield=idvideo&fieldid=64&parentid=59 so I know my php generating the xml is fine.
Thanks!!
Why are you reinventing the wheel? If you're using mootools, making an ajax request is extremely easy (docs, referring to newest version, but in this case Request hasn't changed):
new Request({
url : './gdwcommreply_genxml.php?getfield=idvideo&fieldid=64&parentid=59',
onSuccess : function(responseText, responseXML){
/* here, do stuff with your response */
},
onFailure : function(xhr){
/* the XMLHttpRequest instance. */
}
}).send();
Then, are you sure the url is correct?