I have successfully used await browser.tabs.sendMessage in chrome to get response from the listener, but the same code in firefox does not work. await browser.tabs.sendMessage return immediately and sets response to undefined. In content script inject.js, sendResponse should be called after 1000ms timeout.
I attached a minimalistic example. Any idea why await browser.tabs.sendMessage
returns what sendResponse set only in chrome, but not in firefox?
//inject.js
(async () => {
if (typeof browser === "undefined") {
var browser = chrome;
}
browser.runtime.onMessage.addListener((msg, sender, sendResponse) => {
console.log(msg);
setTimeout(function(){
let pageObject = {a:1};
sendResponse(pageObject);
},1000)
return true;
});
})();
//background.js
(async () => {
if (typeof browser === "undefined") {
var browser = chrome;
}
//**code for injecting content scripts on extension reload**
browser.runtime.onInstalled.addListener(async () => {
let manifest = browser.runtime.getManifest();
for (const cs of manifest.content_scripts) {
for (const tab of await browser.tabs.query({ url: cs.matches })) {
browser.scripting.executeScript({
target: { tabId: tab.id },
files: cs.js,
});
}
}
});
async function SendMessageToFront(message) {
let resolve;
const promise = new Promise(r => resolve = r);
browser.tabs.query({}, async function (tabs) {
for (let index = 0; index < tabs.length; index++) {
const tab = tabs[index];
if (tab.url) {
let url = new URL(tab.url)
if (url.hostname.includes("tragetdomain.com")) {
var startTime = performance.now()
let response = await browser.tabs.sendMessage(tab.id, { message: message });
var endTime = performance.now()
console.log(`Call to doSomething took ${endTime - startTime} milliseconds`) // this takes 0ms
console.log("got response");
console.log(response); // this is undefined
console.log(browser.runtime.lastError); // this is empty
resolve(response);
break;
}
}
}
});
return promise;
}
await SendMessageToFront();
})();
I guess for the tests in firefox you do the reload of the background script (F5 or the specific button in devtools)
Just as you have coded the background you have little hope of getting an answer because every time you reload the background you break the wire with all content scripts injected into the page(s).
Move the browser check inside the "SendMessageToFront" function. Move the "SendMessageToFront" function (async is not needed) to the main thread and run that function in the main thread.
/*async*/ function SendMessageToFront(message) {
if (typeof browser === "undefined")
var browser = chrome;
let resolve;
const promise = new Promise(r => resolve = r);
browser.tabs.query({}, async function(tabs) {
for (let index = 0; index < tabs.length; index++) {
const tab = tabs[index];
if (tab.url) {
let url = new URL(tab.url);
if (url.hostname.includes("tragetdomain.com")) {
var startTime = performance.now()
let response = await browser.tabs.sendMessage(tab.id, {'message': message});
var endTime = performance.now()
console.log(`Call to doSomething took ${endTime - startTime} milliseconds`) // this takes 0ms
console.log("got response");
console.log(response); // this is undefined
console.log(browser.runtime.lastError); // this is empty
resolve(response);
break
}
}
}
});
return promise
}
(async _ => {
await SendMessageToFront()
})();
in this way you will get an error message as soon as the background is ready which tells you that the content script on the other side does not exists or it's not ready yet, but now, when the content script will be ready, you should just re-launch the function from the background script devtools
(async _ => {
await SendMessageToFront()
})();
this time you will get the correct answer {a: 1}
I can't seem to be able to display a response from my custom .proc file. I'm running php-phantomjs on laravel 5.6 on ubuntu. I've got a URL endpoint I'm using to invoke the custom file as follows:
public function getLinks()
{
$location = base_path('phantom-scripts');
$serviceContainer = ServiceContainer::getInstance();
$procedureLoader = $serviceContainer->get('procedure_loader_factory')
->createProcedureLoader($location);
$client = Client::getInstance();
$client->getEngine()->setPath(\Config::get('phantomPath'));
$client->setProcedure('my_procedure');
$client->getProcedureLoader()->addLoader($procedureLoader);
$request = $client->getMessageFactory()->createRequest();
$response = $client->getMessageFactory()->createResponse();
$client->send($request, $response);
dd($response->getContent());
}
Here's the .proc file
var page = require('webpage').create();
var fs = require('fs');
console.log("loading...");
var url = 'http://www.example.com';
page.open(url, function (status) {
if (status === "success") {
var contentNeeded = page.evaluate(function () {
return document.querySelector("h1").outerHTML;
});
console.log(contentNeeded);
}
});
phantom.exit();
I'd like to display the 'contentNeeded' on my browser.
PhantomJS exits too early in your script.
page.open calls are asynchronous (who knows how much time opening that url will take), so phantom.exit() runs right after the browser's only started opening that url. Instead exit PhantomJS after getting the info, in callback:
if (status === "success") {
var contentNeeded = page.evaluate(function () {
return document.querySelector("h1").outerHTML;
});
console.log(contentNeeded);
phantom.exit();
}
The data on the webpage is displayed dynamically and it seems that checking for every change in the html and extracting the data is a very daunting task and also needs me to use very unreliable XPaths. So I would want to be able to extract the data from the XHR packets.
I hope to be able to extract information from XHR packets as well as generate 'XHR' packets to be sent to the server.
The extracting information part is more important for me because the sending of information can be handled easily by automatically triggering html elements using casperjs.
I'm attaching a screenshot of what I mean.
The text in the response tab is the data I need to process afterwards. (This XHR response has been received from the server.)
This is not easily possible, because the resource.received event handler only provides meta data like url, headers or status, but not the actual data. The underlying phantomjs event handler acts the same way.
Stateless AJAX Request
If the ajax call is stateless, you may repeat the request
casper.on("resource.received", function(resource){
// somehow identify this request, here: if it contains ".json"
// it also also only does something when the stage is "end" otherwise this would be executed two times
if (resource.url.indexOf(".json") != -1 && resource.stage == "end") {
var data = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, resource.url);
// do something with data, you might need to JSON.parse(data)
}
});
casper.start(url); // your script
You may want to add the event listener to resource.requested. That way you don't need to way for the call to complete.
You can also do this right inside of the control flow like this (source: A: CasperJS waitForResource: how to get the resource i've waited for):
casper.start(url);
var res, resData;
casper.waitForResource(function check(resource){
res = resource;
return resource.url.indexOf(".json") != -1;
}, function then(){
resData = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, res.url);
// do something with the data here or in a later step
});
casper.run();
Stateful AJAX Request
If it is not stateless, you would need to replace the implementation of XMLHttpRequest. You will need to inject your own implementation of the onreadystatechange handler, collect the information in the page window object and later collect it in another evaluate call.
You may want to look at the XHR faker in sinon.js or use the following complete proxy for XMLHttpRequest (I modeled it after method 3 from How can I create a XMLHttpRequest wrapper/proxy?):
function replaceXHR(){
(function(window, debug){
function args(a){
var s = "";
for(var i = 0; i < a.length; i++) {
s += "\t\n[" + i + "] => " + a[i];
}
return s;
}
var _XMLHttpRequest = window.XMLHttpRequest;
window.XMLHttpRequest = function() {
this.xhr = new _XMLHttpRequest();
}
// proxy ALL methods/properties
var methods = [
"open",
"abort",
"setRequestHeader",
"send",
"addEventListener",
"removeEventListener",
"getResponseHeader",
"getAllResponseHeaders",
"dispatchEvent",
"overrideMimeType"
];
methods.forEach(function(method){
window.XMLHttpRequest.prototype[method] = function() {
if (debug) console.log("ARGUMENTS", method, args(arguments));
if (method == "open") {
this._url = arguments[1];
}
return this.xhr[method].apply(this.xhr, arguments);
}
});
// proxy change event handler
Object.defineProperty(window.XMLHttpRequest.prototype, "onreadystatechange", {
get: function(){
// this will probably never called
return this.xhr.onreadystatechange;
},
set: function(onreadystatechange){
var that = this.xhr;
var realThis = this;
that.onreadystatechange = function(){
// request is fully loaded
if (that.readyState == 4) {
if (debug) console.log("RESPONSE RECEIVED:", typeof that.responseText == "string" ? that.responseText.length : "none");
// there is a response and filter execution based on url
if (that.responseText && realThis._url.indexOf("whatever") != -1) {
window.myAwesomeResponse = that.responseText;
}
}
onreadystatechange.call(that);
};
}
});
var otherscalars = [
"onabort",
"onerror",
"onload",
"onloadstart",
"onloadend",
"onprogress",
"readyState",
"responseText",
"responseType",
"responseXML",
"status",
"statusText",
"upload",
"withCredentials",
"DONE",
"UNSENT",
"HEADERS_RECEIVED",
"LOADING",
"OPENED"
];
otherscalars.forEach(function(scalar){
Object.defineProperty(window.XMLHttpRequest.prototype, scalar, {
get: function(){
return this.xhr[scalar];
},
set: function(obj){
this.xhr[scalar] = obj;
}
});
});
})(window, false);
}
If you want to capture the AJAX calls from the very beginning, you need to add this to one of the first event handlers
casper.on("page.initialized", function(resource){
this.evaluate(replaceXHR);
});
or evaluate(replaceXHR) when you need it.
The control flow would look like this:
function replaceXHR(){ /* from above*/ }
casper.start(yourUrl, function(){
this.evaluate(replaceXHR);
});
function getAwesomeResponse(){
return this.evaluate(function(){
return window.myAwesomeResponse;
});
}
// stops waiting if window.myAwesomeResponse is something that evaluates to true
casper.waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
});
casper.run();
As described above, I create a proxy for XMLHttpRequest so that every time it is used on the page, I can do something with it. The page that you scrape uses the xhr.onreadystatechange callback to receive data. The proxying is done by defining a specific setter function which writes the received data to window.myAwesomeResponse in the page context. The only thing you need to do is retrieving this text.
JSONP Request
Writing a proxy for JSONP is even easier, if you know the prefix (the function to call with the loaded JSON e.g. insert({"data":["Some", "JSON", "here"],"id":"asdasda")). You can overwrite insert in the page context
after the page is loaded
casper.start(url).then(function(){
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
or before the request is received (if the function is registered just before the request is invoked)
casper.on("resource.requested", function(resource){
// filter on the correct call
if (resource.url.indexOf(".jsonp") != -1) {
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}
}).run();
casper.start(url).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
I may be late into the party, but the answer may help someone like me who would fall into this problem later in future.
I had to start with PhantomJS, then moved to CasperJS but finally settled with SlimerJS. Slimer is based on Phantom, is compatible with Casper, and can send you back the response body using the same onResponseReceived method, in "response.body" part.
Reference: https://docs.slimerjs.org/current/api/webpage.html#webpage-onresourcereceived
#Artjom's answer's doesn't work for me in the recent Chrome and CasperJS versions.
Based on #Artjom's answer and based on gilly3's answer on how to replace XMLHttpRequest, I have composed a new solution that should work in most/all versions of the different browsers. Works for me.
SlimerJS cannot work on newer version of FireFox, therefore no good for me.
Here is the the generic code to add a listner to load of XHR (not dependent on CasperJS):
var addXHRListener = function (XHROnStateChange) {
var XHROnLoad = function () {
if (this.readyState == 4) {
XHROnStateChange(this)
}
}
var open_original = XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open = function (method, url, async, unk1, unk2) {
this.requestUrl = url
open_original.apply(this, arguments);
};
var xhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function () {
var xhr = this;
if (xhr.addEventListener) {
xhr.removeEventListener("readystatechange", XHROnLoad);
xhr.addEventListener("readystatechange", XHROnLoad, false);
} else {
function readyStateChange() {
if (handler) {
if (handler.handleEvent) {
handler.handleEvent.apply(xhr, arguments);
} else {
handler.apply(xhr, arguments);
}
}
XHROnLoad.apply(xhr, arguments);
setReadyStateChange();
}
function setReadyStateChange() {
setTimeout(function () {
if (xhr.onreadystatechange != readyStateChange) {
handler = xhr.onreadystatechange;
xhr.onreadystatechange = readyStateChange;
}
}, 1);
}
var handler;
setReadyStateChange();
}
xhrSend.apply(xhr, arguments);
};
}
Here is CasperJS code to emit a custom event on load of XHR:
casper.on("page.initialized", function (resource) {
var emitXHRLoad = function (xhr) {
window.callPhantom({eventName: 'xhr.load', eventData: xhr})
}
this.evaluate(addXHRListener, emitXHRLoad);
});
casper.on('remote.callback', function (data) {
casper.emit(data.eventName, data.eventData)
});
Here is a code to listen to "xhr.load" event and get the XHR response body:
casper.on('xhr.load', function (xhr) {
console.log('xhr load', xhr.requestUrl)
console.log('xhr load', xhr.responseText)
});
Additionally, you can also directly download the content and manipulate it later.
Here is the example of the script I am using to retrieve a JSON and save it locally :
var casper = require('casper').create({
pageSettings: {
webSecurityEnabled: false
}
});
var url = 'https://twitter.com/users/username_available?username=whatever';
casper.start('about:blank', function() {
this.download(url, "hop.json");
});
casper.run(function() {
this.echo('Done.').exit();
});
I have the following function in my controller:
$scope.model.listApplicantStatuses = function(){
var statusChoices = jobsService.getApplicantStatuses();
if(statusChoices !== null)
return statusChoices;
//if($scope.model.listApplicantStatuses.inProgress)
// return;
//$scope.model.listApplicantStatuses.inProgress = true;
jobsService.fetchApplicantStatuses().then(function(data){
jobsService.setApplicantStatuses(data.data);
return data.data;
},
function(data){
$scope.layout.showNotification('error', 10 * 1000, 'we are is experiencing technical difficulties Contact Support');
});
}
corresponding service code:
jobsServ.fetchApplicantStatuses = function(){
return $http.get(utils.getBaseUrl() + '/applications/status_choices', utils.getConfig());
}
jobsServ.getApplicantStatuses = function(){
return that.applicantStatusChoices;
},
jobsServ.setApplicantStatuses = function(choices){
that.applicantStatusChoices = choices;
},
Example DOM usage:
<select class="statusSelect" data-ng-model="model.editedApplicant[applicant.id].status" data-ng-show="layout.statusVisible(applicant.id) && !layout.statusLoader[applicant.id]" data-ng-options="key as val for (key, val) in model.listApplicantStatuses()" data-ng-change="model.updateStatus(applicant.id)"></select>
Now, the problem that I am having is that while chrome waits until the first function call completes, and then gives me the data that I get from the AJAX call, and returns undefined in the meanwhile, Firefox calls the function over and over again, creating a whole lot of uneeded XHR requests.
I commented out the code that was setting the inProgress $scope variable, a because it seems to much of a jQurish solution to me, and because that would force me to change my code in many places, and create another Boolean flag such as this per every request to the server.
I would expect the Firefox behaviour. Perhaps you should change the logic as:
// a variable to keep statuses, or a promise
$scope.applicantStatusList = $scope.model.listApplicantStatuses();
Change listApplicantStatuses() to return the promise:
$scope.model.listApplicantStatuses = function() {
var statusChoices = jobsService.getApplicantStatuses();
if(statusChoices !== null)
return statusChoices;
return jobsService.fetchApplicantStatuses().then(function(data){
jobsService.setApplicantStatuses(data.data);
return data.data;
},
function(data){
$scope.layout.showNotification(...);
});
};
And use the applicantStatusList in the <select>:
<select ... data-ng-options="key as val for (key, val) in applicantStatusList" ...></select>
I solved this by allocating a local variable named _root at the function that sends the xhr, setting _root.ingProgress to true once the request is send, and switching it to false once an answer is received.
Works like a charm.
code:
$scope.model.listApplicantStatuses = function(){
var statusChoices = jobsService.getApplicantStatuses();
if(statusChoices !== null)
return statusChoices;
var _root = this;
if(_root.inProgress)
return;
_root.inProgress = true;
jobsService.fetchApplicantStatuses().then(function(data){
jobsService.setApplicantStatuses(data.data);
_root.inProgress = false;
return data.data;
},
function(data){
_root.inProgress = false;
$scope.layout.showNotification('error', 10 * 1000, 'We are currently experiencing technical difficulties Contact Support');
});
}
Hi I can see this has been discussed but after perusing the issues/answers I still don't seem to be able to get even this simple AJAX call to bump out of ready state 1.
Here's the Javascript I have:
<script language="javascript" type="text/javascript">
var request;
function createRequest()
{
try
{
request = new XMLHttpRequest();
} catch (trymicrosoft) {
try {
request = new ActiveXObject("Msxml2.XMLHTTP");
} catch (othermicrosoft) {
try {
request = new ActiveXObject("Microsoft.XMLHTTP");
} catch (failed) {
request = false;
}
}
}
if (!request)
alert("Error initializing XMLHttpRequest!");
}
function loadClassesBySchool()
{
//get require web form pieces for this call
createRequest(); // function to get xmlhttp object
var schoolId = getDDLSelectionValue("ddlSchools");
var grade = getDDLSelectionValue("ddlGrades");
var url = "courses.php?grades=" + escape(grade) + "&schoolId=" + escape(schoolId);
//open server connection
request.open("GET", url, true);
//Setup callback function for server response
//+++read on overflow that some fixed the issue with an onload event this simply had
//+++the handle spitback 2 readystate = 1 alerts
request.onload = updateCourses();
request.onreadystatechanged = updateCourses();
//send the result
request.send();
}
function updateCourses()
{
alert('ready state changed' + request.readyState);
}
function getDDLSelectionValue(ddlID)
{
return document.getElementById(ddlID).options[document.getElementById(ddlID).selectedIndex].value;
}
</script>
The PHP is HERE just a simple print which if i navigate to in the browser (IE/Chrome) loads fine:
<?php
print "test";
?>
I'm quite new at this but seems like I can't get the most bare bones AJAX calls to work, any help as to how work past this would be greatly appreciated.
All I get out of my callback function 'updateCourses' is a 1...
Well after more digging I actually gave up and switched over to jQuery which should for all intents and purposes be doing the EXACT same thing except for the fact that jQuery works... I was just less comfortable with it but so be it.
Here's the jQuery to accomplish the same:
function loadCoursesBySchool(){
var grades = getDDLSelectionValue("ddlGrades");
var schoolId = getDDLSelectionValue("ddlSchools");
jQuery.ajax({
url: "courses.php?grades=" + grades + "&schoolId=" + schoolId,
success: function (data) {
courseDisplay(data);
}
});
}
function courseDisplay(response)
{
//check if anything was setn back!?
if(!response)
{
$("#ddlCourses").html("");
//do nothing?
}
else
{
//empty DLL
$("#ddlCourses").html("");
//add entries
$(response).appendTo("#ddlCourses");
}
}