I am using an observer on "http-on-modify-request" to analyze HTTP requests (and responses with the corresponding other observers).
Is it possible to determine whether the HTTP request / response is the main frame loading (the actual page DOM)? As opposed to another resource (image, css, sub_frame, etc.).
The docs have most of the answer you're looking for here and I've modified it below for use with the addon-sdk.
You can watch for an IFRAME by comparing the location with the top.document location.
I don't think there's an easy way to detect loading of images, etc so you'll probably want to just watch for the first hit that's not an IFRAME and regard everything else as css/image/script content loading.
var chrome = require("chrome");
var httpmods = {
observe : function(aSubject, aTopic, aData) {
console.log("observer", aSubject, aTopic, aData);
aSubject.QueryInterface(chrome.Ci.nsIHttpChannel);
var url = aSubject.URI.spec;
var dom = this.getBrowserFromChannel(aSubject);
if (dom) {
if (dom.top.document && dom.location === dom.top.document.location) {
console.log("ISN'T IFRAME");
} else {
console.log("IS IFRAME");
}
}
},
getBrowserFromChannel: function (aChannel) {
try {
var notificationCallbacks =
aChannel.notificationCallbacks ? aChannel.notificationCallbacks : aChannel.loadGroup.notificationCallbacks;
if (!notificationCallbacks)
return null;
var domWin = notificationCallbacks.getInterface(chrome.Ci.nsIDOMWindow);
return domWin;
}
catch (e) {
dump(e + "\n");
return null;
}
}
}
require("observer-service").add("http-on-modify-request", httpmods.observe, httpmods);
Related
I created a script in Google Sheets, which is working well but after a while I'm getting the following error:
Exception: Service invoked too many times for one day: urlfetch
I think I called the function like 200-300 times in the day, for what I checked it should be below the limit.
I read we can use cache to avoid this issue but not sure how to use it in my code.
function scrapercache(url) {
var result = [];
var description;
var options = {
'muteHttpExceptions': true,
'followRedirects': false,
};
var cache = CacheService.getScriptCache();
var properties = PropertiesService.getScriptProperties();
try {
let res = cache.get(url);
if (!res) {
// trim url to prevent (rare) errors
url.toString().trim();
var r = UrlFetchApp.fetch(url, options);
var c = r.getResponseCode();
// check for meta refresh if 200 ok
if (c == 200) {
var html = r.getContentText();
cache.put(url, "cached", 21600);
properties.setProperty(url, html);
var $ = Cheerio.load(html); // make sure this lib is added to your project!
// meta description
if ($('meta[name=description]').attr("content")) {
description = $('meta[name=description]').attr("content").trim();
}
}
result.push([description]);
}
}
catch (error) {
result.push(error.toString());
}
finally {
return result;
}
}
how can I use cache like this to enhance my script please?
var cache = CacheService.getScriptCache();
var result = cache.get(url);
if(!result) {
var response = UrlFetchApp.fetch(url);
result = response.getContentText();
cache.put(url, result, 21600);
Answer:
You can implement CacheService and PropertiesService together and only retrieve the URL again after a specified amount of time.
Code Change:
Be aware that additional calls to retrieving the cache and properties will slow your function down, especially if you are doing this a few hundred times.
As the values of the cache can be a maximum of 100 KB, we will use CacheService to keep track of which URLs are to be retrieved, but PropertiesService to store the data.
You can edit your try block as so:
var cache = CacheService.getScriptCache();
var properties = PropertiesService.getScriptProperties();
try {
let res = cache.get(url);
if (!res) {
// trim url to prevent (rare) errors
url.toString().trim();
var r = UrlFetchApp.fetch(url, options);
var c = r.getResponseCode();
// check for meta refresh if 200 ok
if (c == 200) {
var html = r.getContentText();
cache.put(url, "cached", 21600);
properties.setProperty(url, html);
var $ = Cheerio.load(html); // make sure this lib is added to your project!
// meta description
if ($('meta[name=description]').attr("content")) {
description = $('meta[name=description]').attr("content").trim();
}
}
result.push([description]);
}
}
catch (error) {
result.push(error.toString());
}
finally {
return result;
}
References:
Class CacheService | Apps Script | Google Developers
Class Cache | Apps Script | Google Developers
Class PropertiesService | Apps Script | Google Developers
Related Questions:
Service invoked too many times for one day: urlfetch
i am using ajax and jquery to send a synchronous http request.
I have to use synchronous request because i want to return some value from ajax function after evaluating result of the server side script.
i know synchronous request will freeze the browser but due to my requirement i will have to do this request.
i also knows that in synchronous request there is no use of onreadystatechange function we should evaluate our result after sending the request or below the send function.By doing this my code is in working state.
But, my problem is that when i used onreadystatechange function it is working in firefox >=4 version but not working in below firefox 4 version.
Please. help me in finding out whether problem is with the code or the browser.
i am not able to find out the bug now i am helpless...plz help
here is my code
function test(txt_obj) {
var keywords = txt_obj.value;
var SHttpRequestObject = false;
var url = "/speller/server-scripts/ifmisspelled_words.html" + '?keywords=' + keywords);
var speller_res = 0;
if (window.XMLHttpRequest){
SHttpRequestObject = new XMLHttpRequest();
} else if (window.ActiveXObject){
SHttpRequestObject = new ActiveXObject("Microsoft.XMLHTTP");
}
SHttpRequestObject.open("POST", url, false);
if (SHttpRequestObject){
SHttpRequestObject.onreadystatechange = function() {
if (SHttpRequestObject.readyState == 4 && SHttpRequestObject.status == 200)
{
var result = eval("(" + SHttpRequestObject.responseText + ")");
if(result.error) {
speller_res = 1;
} else if(result.word_exist) {
speller_res = 1;
}
else if(result.word_not_exist) {
speller_res = 0;
}
}
};
}
SHttpRequestObject.send(null);
return speller_res;
}
From MDN:
onreadystatechange
Warning: This must not be used from native code. You should also not
use this with synchronous requests.
I'm using LESS CSS (more exactly less.js) which seems to exploit LocalStorage under the hood. I had never seen such an error like this before while running my app locally, but now I get "Persistent storage maximum size reached" at every page display, just above the link the unique .less file of my app.
This only happens with Firefox 12.0 so far.
Is there any way to solve this?
P.S.: mainly inspired by Calculating usage of localStorage space, this is what I ended up doing (this is based on Prototype and depends on a custom trivial Logger class, but this should be easily adapted in your context):
"use strict";
var LocalStorageChecker = Class.create({
testDummyKey: "__DUMMY_DATA_KEY__",
maxIterations: 100,
logger: new Logger("LocalStorageChecker"),
analyzeStorage: function() {
var result = false;
if (Modernizr.localstorage && this._isLimitReached()) {
this._clear();
}
return result;
},
_isLimitReached: function() {
var localStorage = window.localStorage;
var count = 0;
var limitIsReached = false;
do {
try {
var previousEntry = localStorage.getItem(this.testDummyKey);
var entry = (previousEntry == null ? "" : previousEntry) + "m";
localStorage.setItem(this.testDummyKey, entry);
}
catch(e) {
this.logger.debug("Limit exceeded after " + count + " iteration(s)");
limitIsReached = true;
}
}
while(!limitIsReached && count++ < this.maxIterations);
localStorage.removeItem(this.testDummyKey);
return limitIsReached;
},
_clear: function() {
try {
var localStorage = window.localStorage;
localStorage.clear();
this.logger.debug("Storage clear successfully performed");
}
catch(e) {
this.logger.error("An error occurred during storage clear: ");
this.logger.error(e);
}
}
});
document.observe("dom:loaded",function() {
var checker = new LocalStorageChecker();
checker.analyzeStorage();
});
P.P.S.: I didn't measure the performance impact on the UI yet, but a decorator could be created and perform the storage test only every X minutes (with the last timestamp of execution in the local storage for instance).
Here is a good resource for the error you are running into.
http://www.sitepoint.com/building-web-pages-with-local-storage/#fbid=5fFWRXrnKjZ
Gives some insight that localstorage only has so much room and you can max it out in each browser. Look into removing some data from localstorage to resolve your problem.
Less.js persistently caches content that is #imported. You can use this script to clear content that is cached. Using the script below you can call the function destroyLessCache('/path/to/css/') and it will clear your localStorage of css files that have been cached.
function destroyLessCache(pathToCss) { // e.g. '/css/' or '/stylesheets/'
if (!window.localStorage || !less || less.env !== 'development') {
return;
}
var host = window.location.host;
var protocol = window.location.protocol;
var keyPrefix = protocol + '//' + host + pathToCss;
for (var key in window.localStorage) {
if (key.indexOf(keyPrefix) === 0) {
delete window.localStorage[key];
}
}
}
I would like to write a little component with XPCOM that can sniff all HTTP responses received by the browser. Right now the only examples that I can find (like the one appended below) only allow me to retrieve the response for a request that I fire myself:
var req = new XMLHttpRequest();
req.open('GET', 'http://www.mozilla.org/', true);
req.onreadystatechange = function (aEvt) {
if (req.readyState == 4) {
if(req.status == 200)
dump(req.responseText);
else
dump("Error loading page\n");
}
};
What I want is for any HTTP response that the browser receives get the HTTP headers of the corresponding request.
Thanks
You can also use the http-on-modify-request and http-on-examine-response notifications. See https://developer.mozilla.org/en/XUL_School/Intercepting_Page_Loads#HTTP_Observers
You can sniff all http traffic via the nsIHttpActivityObserver, an example cribbed from the Firefox web console:
const Cc = Components.classes;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyServiceGetter(this, "activityDistributor",
"#mozilla.org/network/http-activity-distributor;1",
"nsIHttpActivityDistributor");
let httpTrafficObserver = {
/**
* Begin observing HTTP traffic that we care about,
* namely traffic that originates inside any context that a Heads Up Display
* is active for.
*/
startHTTPObservation: function httpObserverFactory()
{
// creates an observer for http traffic
var self = this;
var httpObserver = {
observeActivity :
function observeActivity(aChannel,
aActivityType,
aActivitySubtype,
aTimestamp,
aExtraSizeData,
aExtraStringData)
{
if (aActivityType ==
activityDistributor.ACTIVITY_TYPE_HTTP_TRANSACTION ||
aActivityType ==
activityDistributor.ACTIVITY_TYPE_SOCKET_TRANSPORT) {
aChannel = aChannel.QueryInterface(Ci.nsIHttpChannel);
let transCodes = this.httpTransactionCodes;
if (aActivitySubtype ==
activityDistributor.ACTIVITY_SUBTYPE_REQUEST_HEADER ) {
let httpActivity = {
url: aChannel.URI.spec,
method: aChannel.requestMethod,
channel: aChannel
};
}
}
},
httpTransactionCodes: {
0x5001: "REQUEST_HEADER",
0x5002: "REQUEST_BODY_SENT",
0x5003: "RESPONSE_START",
0x5004: "RESPONSE_HEADER",
0x5005: "RESPONSE_COMPLETE",
0x5006: "TRANSACTION_CLOSE",
0x804b0003: "STATUS_RESOLVING",
0x804b0007: "STATUS_CONNECTING_TO",
0x804b0004: "STATUS_CONNECTED_TO",
0x804b0005: "STATUS_SENDING_TO",
0x804b000a: "STATUS_WAITING_FOR",
0x804b0006: "STATUS_RECEIVING_FROM"
}
};
this.httpObserver = httpObserver;
activityDistributor.addObserver(httpObserver);
}
};
and http://mxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsIHttpActivityObserver.idl
for (var i = 0; i < 5; ++i) {
var xhr;
if (window.XMLHttpRequest) {
xhr = new XMLHttpRequest();
} else if (window.ActiveXObject) {
xhr = new ActiveXObject("Msxml2.XMLHTTP");
}
xhr.open('GET', '/Test/LongOperation?p=' + new Date());
xhr.send('');
}
This is only a demo (not live code) but it illustrates the core problem.
LongOperation is a method that returns a result after 10 seconds.
Questions:
Why does IE8 (and maybe other IEs) hang when the user tries to navigate away from page right after the above code snippet has been executed? FireFox/Safari cancel these requests and allow navigation to another page. If you replace 'i < 5' with 'i < 4' then IE would not hang.
How to work around this ugly IE behavior? Users are very upset when their browser suddenly hangs.
Most browsers have an inbuilt limit of 4 connections to any given server. One way to work around this "problem" might be to use a different hostname for out of band XML requests - your user requests will go to the main hosts, while the AJAX requests can go to the second server.
My answer to my question. I abort all not completed xhr objects in window.onbeforeunload. At least this solution works for me. I slightly override $.ajax() method behavior:
;(function($) {
var rq = [];
var ajax = $.ajax;
$.ajax = function(settings) {
// override complete() operation
var complete = settings.complete;
settings.complete = function(xhr) {
if (xhr) {
// xhr may be undefined, for example when downloading JavaScript
for (var i = 0, len = rq.length; i < len; ++i) {
if (rq[i] == xhr) {
// drop completed xhr from list
rq.splice(i, 1);
break;
}
}
}
// execute base
if (complete) {
complete.apply(this, arguments)
}
}
var r = ajax.apply(this, arguments);
if (r) {
// r may be undefined, for example when downloading JavaScript
rq.push(r);
}
return r;
};
// 'kill' all pending xhrs
$(window).bind('beforeunload', function() {
$.each(rq, function(i, xhr) {
try {
xhr.abort();
} catch(e) {
$debug.fail('failed to abort xhr');
}
});
rq = [];
});
})(jQuery);
$debug - my utility class
Try running them asynchronously and then triggering the next http request when the each completes. I suspect that the xmlhttp request is blocking the UI thread of IE whereas the implementations of that on other browsers is a little more graceful.
Hopefully that will give you a work-around for question 2 but I can only guess at the true reason for question 1, it could just be a bug.