I'm using PhantomJS + d3 to render a map of US Zipcodes as a backend process. The rendering and zip code counting takes long enough that putting the html and d3 js in the browser would require a minute to load and caused other issues, so we moved it to the backend.
If I send one request via curl to the node server that PhantomJS starts up, no problem. If I space our multiple map requests by about 15 seconds in between, also no problem. However, if I launch a couple curl requests very quickly, the rendered images wind up looking the same (aka the same image is written to multiple files.) Here is the phantom script:
var port,
server,
service,
page,
url,
svgDrawer;
fs = require('fs');
port = 9494;
server = require('webserver').create();
page = require('webpage').create();
service = server.listen(port, function (request, response) {
var drawerPayload = null;
try{
drawerPayload=JSON.parse(request.post);
} catch(e){
response.statusCode = 417;
response.write("Error : Invalid Input JSON");
response.close();
return;
}
url = 'file:///' + fs.absolute('./'+drawerPayload.inFile);
page.open(url, function (status) {
if(status=="success"){
page.evaluate(function(data){
$("body").on( "click", data, chartBuilder );
$("body").click();
var maxtimeOutMillis = 15000,
start = new Date().getTime(),
condition = false,
interval = setInterval(function() {
if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
condition = $("svg.chart").hasClass("done"); //< defensive code
} else {
if(!condition) {
clearInterval(interval)
} else {
page.render(drawerPayload.outFile);
clearInterval(interval); //< Stop this interval
}
}
}, 250); //< repeat check every 250ms
});
response.statusCode = 200;
} else {
response.statusCode = 404;
response.write("Not Found"+url);
}
response.close();
return;
});
page.onError = function (msg, trace) {
console.log(msg);
trace.forEach(function(item) {
console.log(' ', item.file, ':', item.line);
})
response.statusCode = 417;
response.write("Error : "+msg);
response.close();
return;
}
});
and the html+d3 looks like this:
<!DOCTYPE html>
<meta charset="utf-8">
<style>
.zip {
stroke: none;
}
.chart {
fill: white;
width: 1000px;
height: 500px;
}
</style>
<body>
<div id="chart-container">
<svg class="chart"></svg>
</div>
</body>
<script src="./jquery-min.js"></script>
<script src="./d3.min.js"></script>
<script type="text/javascript" src="http://d3js.org/topojson.v1.min.js"></script>
<script>
function chartBuilder(e){
var zip_data = e.data;
$.getJSON("zips_us_topo.json", function(us){
console.log("rendering...\n");
var width = 1000;
var height = 500;
var projection = d3.geo.albersUsa()
.scale(width)
.translate([width / 2, height / 2]);
var path = d3.geo.path().projection(projection);
var color = d3.scale.log().domain([1,zip_data.max+1]).range(["#cccccc","#f63337"]);
var svg = d3.select("svg.chart")
.attr("width", width)
.attr("height", height)
.style({margin: "10px 100px"})
.append("g")
.attr("class", "counties")
.selectAll("path")
.data(topojson.feature(us, us.objects.zip_codes_for_the_usa).features)
.enter()
.append("path")
.attr("class", "zip")
.style({fill: function(d){
return color(zip_data.counts[d.properties.zip] ? zip_data.counts[d.properties.zip]+1 : 1);
}})
.attr("d", path);
svg.classed("done", true);
});
}
</script>
If the requests are all curled at the same time, it looks like it's writing one image to all of the output files. Does PhantomJS create a new page for each request, or is it loading the same request each time?
You have only a single page instance for all requests that come. This might create some race conditions when new requests come in and hijack the current page.open() request. There are basically two ways to solve this depending on your preferred scenario.
Multiple "tabs"
The simple fix would be to create a new page instance for every request and they will be essentially different tabs in the same browser. So if cookies or localStorage is an issue, this is not for you.
Move page = require('webpage').create(); inside the server.listen callback and don't forget to close() the page instance after use.
Only one request at a time
Since this is a not so short running process, you can start a page.open() when it is not currently running and put all incoming requests into a queue as long as page.open() hasn't finished. After it is finished, save the response, go through the request queue and respond with the same response to all of them.
This is of course much nicer on memory consumption than the first solution if there really are many concurrent requests.
There are other problems with your code though. You use setInterval() inside page.evaluate(), which breaks off of the control flow. response.statusCode = 200; will be most likely set before the page is rendered.
That page.render() inside of page.evaluate() is the other problem. page.evaluate() is the sandboxed page context. It doesn't have access to variables defined outside of it and that includes page and require. (Solution for this isolated problem)
Those two problems can be solved with a single blow by waiting outside of the page context for a render condition inside of it. I suggest using waitFor from the examples:
if(status=="success"){
page.evaluate(function(data){
window._finishIndicationVariable = false;
$("body").on( "click", data, chartBuilder );
$("body").click();
var maxtimeOutMillis = 15000,
start = new Date().getTime(),
condition = false,
interval = setInterval(function() {
if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
condition = $("svg.chart").hasClass("done"); //< defensive code
} else {
if(!condition) {
clearInterval(interval)
} else {
window._finishIndicationVariable = true;
clearInterval(interval); //< Stop this interval
}
}
}, 250); //< repeat check every 250ms
});
waitFor(function check(){
return page.evaluate(function(){
return window._finishIndicationVariable;
});
}, function onReady(){
page.render(drawerPayload.outFile);
response.statusCode = 200;
response.close();
});
} else {
response.statusCode = 404;
response.write("Not Found"+url);
response.close();
}
Note that response.close(); is used two times, because one of the if branches is asynchronous and the one is not.
The data on the webpage is displayed dynamically and it seems that checking for every change in the html and extracting the data is a very daunting task and also needs me to use very unreliable XPaths. So I would want to be able to extract the data from the XHR packets.
I hope to be able to extract information from XHR packets as well as generate 'XHR' packets to be sent to the server.
The extracting information part is more important for me because the sending of information can be handled easily by automatically triggering html elements using casperjs.
I'm attaching a screenshot of what I mean.
The text in the response tab is the data I need to process afterwards. (This XHR response has been received from the server.)
This is not easily possible, because the resource.received event handler only provides meta data like url, headers or status, but not the actual data. The underlying phantomjs event handler acts the same way.
Stateless AJAX Request
If the ajax call is stateless, you may repeat the request
casper.on("resource.received", function(resource){
// somehow identify this request, here: if it contains ".json"
// it also also only does something when the stage is "end" otherwise this would be executed two times
if (resource.url.indexOf(".json") != -1 && resource.stage == "end") {
var data = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, resource.url);
// do something with data, you might need to JSON.parse(data)
}
});
casper.start(url); // your script
You may want to add the event listener to resource.requested. That way you don't need to way for the call to complete.
You can also do this right inside of the control flow like this (source: A: CasperJS waitForResource: how to get the resource i've waited for):
casper.start(url);
var res, resData;
casper.waitForResource(function check(resource){
res = resource;
return resource.url.indexOf(".json") != -1;
}, function then(){
resData = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, res.url);
// do something with the data here or in a later step
});
casper.run();
Stateful AJAX Request
If it is not stateless, you would need to replace the implementation of XMLHttpRequest. You will need to inject your own implementation of the onreadystatechange handler, collect the information in the page window object and later collect it in another evaluate call.
You may want to look at the XHR faker in sinon.js or use the following complete proxy for XMLHttpRequest (I modeled it after method 3 from How can I create a XMLHttpRequest wrapper/proxy?):
function replaceXHR(){
(function(window, debug){
function args(a){
var s = "";
for(var i = 0; i < a.length; i++) {
s += "\t\n[" + i + "] => " + a[i];
}
return s;
}
var _XMLHttpRequest = window.XMLHttpRequest;
window.XMLHttpRequest = function() {
this.xhr = new _XMLHttpRequest();
}
// proxy ALL methods/properties
var methods = [
"open",
"abort",
"setRequestHeader",
"send",
"addEventListener",
"removeEventListener",
"getResponseHeader",
"getAllResponseHeaders",
"dispatchEvent",
"overrideMimeType"
];
methods.forEach(function(method){
window.XMLHttpRequest.prototype[method] = function() {
if (debug) console.log("ARGUMENTS", method, args(arguments));
if (method == "open") {
this._url = arguments[1];
}
return this.xhr[method].apply(this.xhr, arguments);
}
});
// proxy change event handler
Object.defineProperty(window.XMLHttpRequest.prototype, "onreadystatechange", {
get: function(){
// this will probably never called
return this.xhr.onreadystatechange;
},
set: function(onreadystatechange){
var that = this.xhr;
var realThis = this;
that.onreadystatechange = function(){
// request is fully loaded
if (that.readyState == 4) {
if (debug) console.log("RESPONSE RECEIVED:", typeof that.responseText == "string" ? that.responseText.length : "none");
// there is a response and filter execution based on url
if (that.responseText && realThis._url.indexOf("whatever") != -1) {
window.myAwesomeResponse = that.responseText;
}
}
onreadystatechange.call(that);
};
}
});
var otherscalars = [
"onabort",
"onerror",
"onload",
"onloadstart",
"onloadend",
"onprogress",
"readyState",
"responseText",
"responseType",
"responseXML",
"status",
"statusText",
"upload",
"withCredentials",
"DONE",
"UNSENT",
"HEADERS_RECEIVED",
"LOADING",
"OPENED"
];
otherscalars.forEach(function(scalar){
Object.defineProperty(window.XMLHttpRequest.prototype, scalar, {
get: function(){
return this.xhr[scalar];
},
set: function(obj){
this.xhr[scalar] = obj;
}
});
});
})(window, false);
}
If you want to capture the AJAX calls from the very beginning, you need to add this to one of the first event handlers
casper.on("page.initialized", function(resource){
this.evaluate(replaceXHR);
});
or evaluate(replaceXHR) when you need it.
The control flow would look like this:
function replaceXHR(){ /* from above*/ }
casper.start(yourUrl, function(){
this.evaluate(replaceXHR);
});
function getAwesomeResponse(){
return this.evaluate(function(){
return window.myAwesomeResponse;
});
}
// stops waiting if window.myAwesomeResponse is something that evaluates to true
casper.waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
});
casper.run();
As described above, I create a proxy for XMLHttpRequest so that every time it is used on the page, I can do something with it. The page that you scrape uses the xhr.onreadystatechange callback to receive data. The proxying is done by defining a specific setter function which writes the received data to window.myAwesomeResponse in the page context. The only thing you need to do is retrieving this text.
JSONP Request
Writing a proxy for JSONP is even easier, if you know the prefix (the function to call with the loaded JSON e.g. insert({"data":["Some", "JSON", "here"],"id":"asdasda")). You can overwrite insert in the page context
after the page is loaded
casper.start(url).then(function(){
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
or before the request is received (if the function is registered just before the request is invoked)
casper.on("resource.requested", function(resource){
// filter on the correct call
if (resource.url.indexOf(".jsonp") != -1) {
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}
}).run();
casper.start(url).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
I may be late into the party, but the answer may help someone like me who would fall into this problem later in future.
I had to start with PhantomJS, then moved to CasperJS but finally settled with SlimerJS. Slimer is based on Phantom, is compatible with Casper, and can send you back the response body using the same onResponseReceived method, in "response.body" part.
Reference: https://docs.slimerjs.org/current/api/webpage.html#webpage-onresourcereceived
#Artjom's answer's doesn't work for me in the recent Chrome and CasperJS versions.
Based on #Artjom's answer and based on gilly3's answer on how to replace XMLHttpRequest, I have composed a new solution that should work in most/all versions of the different browsers. Works for me.
SlimerJS cannot work on newer version of FireFox, therefore no good for me.
Here is the the generic code to add a listner to load of XHR (not dependent on CasperJS):
var addXHRListener = function (XHROnStateChange) {
var XHROnLoad = function () {
if (this.readyState == 4) {
XHROnStateChange(this)
}
}
var open_original = XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open = function (method, url, async, unk1, unk2) {
this.requestUrl = url
open_original.apply(this, arguments);
};
var xhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function () {
var xhr = this;
if (xhr.addEventListener) {
xhr.removeEventListener("readystatechange", XHROnLoad);
xhr.addEventListener("readystatechange", XHROnLoad, false);
} else {
function readyStateChange() {
if (handler) {
if (handler.handleEvent) {
handler.handleEvent.apply(xhr, arguments);
} else {
handler.apply(xhr, arguments);
}
}
XHROnLoad.apply(xhr, arguments);
setReadyStateChange();
}
function setReadyStateChange() {
setTimeout(function () {
if (xhr.onreadystatechange != readyStateChange) {
handler = xhr.onreadystatechange;
xhr.onreadystatechange = readyStateChange;
}
}, 1);
}
var handler;
setReadyStateChange();
}
xhrSend.apply(xhr, arguments);
};
}
Here is CasperJS code to emit a custom event on load of XHR:
casper.on("page.initialized", function (resource) {
var emitXHRLoad = function (xhr) {
window.callPhantom({eventName: 'xhr.load', eventData: xhr})
}
this.evaluate(addXHRListener, emitXHRLoad);
});
casper.on('remote.callback', function (data) {
casper.emit(data.eventName, data.eventData)
});
Here is a code to listen to "xhr.load" event and get the XHR response body:
casper.on('xhr.load', function (xhr) {
console.log('xhr load', xhr.requestUrl)
console.log('xhr load', xhr.responseText)
});
Additionally, you can also directly download the content and manipulate it later.
Here is the example of the script I am using to retrieve a JSON and save it locally :
var casper = require('casper').create({
pageSettings: {
webSecurityEnabled: false
}
});
var url = 'https://twitter.com/users/username_available?username=whatever';
casper.start('about:blank', function() {
this.download(url, "hop.json");
});
casper.run(function() {
this.echo('Done.').exit();
});
I'm currently trying to build my first Chrome extension but I'm having a slight issue with my code.
I want to use XMLHTTPRequest and xpath to display a specific number from an external website as a badge on my icon. The code I'm using in my background.js file is as follows:
var xhr = new XMLHttpRequest();
xhr.open("GET", "http://www.example.com", true);
xhr.onreadystatechange = function() {
if (xhr.readyState == 4) {
// innerText does not let the attacker inject HTML elements.
document.getElementById("resp").innerText = xhr.responseText;
}
}
xhr.send();
var xmlDoc = xhr.responseXML;
xmlDoc.setProperty('SelectionLanguage', 'XPath');
var badgeText = xmldoc.documentElement.selectSingleNode("//[#id='main']/div/div/section/div[1]/div[2]");
chrome.browserAction.setBadgeText({text: badgeText});
chrome.browserAction.setBadgeBackgroundColor({color: "#1f729f"});
I know this code is probably pretty horrible but this is my first extension and I'd really appreciate any help.
Thanks in advance.
It looks like you are expecting some immediate response after xhr.send();
Here the code snipet which is working in one of my extensions (notice this in the callback function):
var xhr = new XMLHttpRequest();
xhr.open("GET", "http://www.example.com", true);
xhr.onreadystatechange = function() {
console.log("XHR callback readyState = " + this.readyState);
if (this.readyState == 4) {
// innerText does not let the attacker inject HTML elements.
document.getElementById("resp").innerText = this.responseText;
var xmlDoc = this.responseXML;
xmlDoc.setProperty('SelectionLanguage', 'XPath');
var badgeText = xmldoc.documentElement.selectSingleNode("//[#id='main']/div/div/section/div[1]/div[2]");
chrome.browserAction.setBadgeText({text: badgeText});
chrome.browserAction.setBadgeBackgroundColor({color: "#1f729f"});
}
}
xhr.send();
This way the code would be executed only after the response is available.
Hope this helps ;-)
I am trying to upload a file by XMLHttpRequest. After uploading I am trying to show preview of the image by calling an action as a source of the image.
Here is the code sample:
var xhr = new XMLHttpRequest();
xhr.file = file;
xhr.onreadystatechange = function (e) {
if (this.readyState == 4 && xhr.status == 200) { // here I am updating the image source
document.getElementById('imagePreview').src = '#Url.Action("GetTPImaegeByName","Techpack",new { area="OMS" })';
}
};
xhr.open('post', 'someurl', false);
var fd = new FormData;
fd.append('photo', file);
xhr.send(fd);
Here #Url.Action("GetTPImaegeByName","Techpack",new { area="OMS" }) returns an image file.
For the first time the source of the image is updating and shown correctly. But if I try again to change the file the it doesn't work. Looks like the action to update the image source is not being called. Need help to solve this problem.
On my site I use one core/frame PHP file. If user hit one of my link (like contact, our about..etc..) the content loaded via ajax. I use the following snippet to achieve this:
var AjaxContent = function(){
var container_div = '';
var content_div = '';
return {
getContent : function(url){
$(container_div).animate({opacity:0},
function(){ // the callback, loads the content with ajax
$(container_div).load(url, //only loads the selected portion
function(){
$(container_div).animate({opacity:1});
}
);
});
},
ajaxify_links: function(elements){
$(elements).click(function(){
AjaxContent.getContent(this.href);
return false;
});
},
init: function(params){
container_div = params.containerDiv;
content_div = params.contentDiv;
return this;
}
}
}();
I need help how to integrate a preloading, so if visitors hit one of my link (for example the gallery menu) will see a little loading image, because now they see the big white nothing for long - long seconds.
Add loading image beforing ajax call and after you get response from server simply replace that image with data like the one below
function(){ // the callback, loads the content with ajax
$(container_div).html("<img src='loading.gif' />");//add image before ajax call
$(container_div).load(url, //only loads the selected portion
function(){
$(container_div).html(data);//replace image with server response data
$(container_div).animate({opacity:1});
}
Try this
var AjaxContent = function(){
var container_div = '';
var content_div = '';
return {
getContent : function(url){
$(container_div).html('Loading...'); //replace with your loading img html code
$(container_div).load(url, //only loads the selected portion
function(){
$(container_div).css({opacity:0});
$(container_div).animate({opacity:1});
});
},
ajaxify_links: function(elements){
$(elements).click(function(){
AjaxContent.getContent(this.href);
return false;
});
},
init: function(params){
container_div = params.containerDiv;
content_div = params.contentDiv;
return this;
}
}
}();