I am trying to cache specific urls and each url has md5 hash and If the urls updated with new md5 i want to remove the current cache and add the new one.
cached url: http://www.mysite.lo/cards/index.php?md5=f51c2ef7795480ef2e0b1bd24c9e07
function shouldFetch(event) {
if ( event.request.url.indexOf( '/cards/') == -1 ) {
return false;
}
return true;
}
self.addEventListener('fetch', function(event) {
if (shouldFetch(event)) {
event.respondWith(
caches.match(event.request).then(function(response) {
if (response !== undefined) {
return response;
} else {
return fetch(event.request).then(function (response) {
let responseClone = response.clone();
caches.open('v1').then(function (cache) {
cache.put(event.request, responseClone);
});
return response;
}).catch(function (err) {
return caches.match(event.request);
});
}
})
);
}
});
I know we can use caches.delete() and so on, but I want to call it only if the md5 updated from the new request.
Thanks
You can accomplish roughly what you describe with the following, which makes use of the ignoreSearch option when calling cache.matchAll():
self.addEventListener('fetch', (event) => {
const CACHE_NAME = '...';
const url = new URL(event.request.url);
if (url.searchParams.has('md5')) {
event.respondWith((async () => {
const cache = await caches.open(CACHE_NAME);
const cachedResponses = await cache.matchAll(url.href, {
// https://developers.google.com/web/updates/2016/09/cache-query-options
ignoreSearch: true,
});
for (const cachedResponse of cachedResponses) {
// If we already have the incoming URL cached, return it.
if (cachedResponse.url === url.href) {
return cachedResponse;
}
// Otherwise, delete the out of date response.
await cache.delete(cachedResponse.url);
}
// If we've gotten this far, then there wasn't a cache match,
// and our old entries have been cleaned up.
const response = await fetch(event.request);
await cache.put(event.request, response.clone());
return response;
})());
}
// Logic for non-md5 use cases goes here.
});
(You could make things slightly more efficient by rearranging some of the cache-manipulation code to bring it out of the critical response path, but that's the basic idea.)
I register '/' (route) .css and .js file as URL that should be cached at first.
But after that I realize that it cached the whole page, which means I don't see any update on my view, event there is an update on database.
So I change it only cache my .css and .js file not route ('/') anymore , I expected that's the problem.
But after awhile, the same problem still occurred. I check on my console it did cache the whole page again, even though my Service Worker file already change like this:
var CACHE_NAME = 'cache-v2';
var urlsToCache = [
'/assets/css/app.css',
'/assets/js/main.js',
'/assets/js/other.js'
];
self.addEventListener('install', function(event) {
event.waitUntil(
caches.open(CACHE_NAME)
.then(function(cache) {
console.log('Opened cache');
return cache.addAll(urlsToCache);
})
);
});
self.addEventListener('activate', function(e) {
console.log('[ServiceWorker] Activate');
e.waitUntil(
caches.keys().then(function(keyList) {
return Promise.all(keyList.map(function(key) {
if (key !== CACHE_NAME) {
console.log('[ServiceWorker] Removing old cache', key);
return caches.delete(key);
}
}));
})
);
return self.clients.claim();
});
self.addEventListener('fetch', function(event) {
event.respondWith(
caches.match(event.request)
.then(function(response) {
// Cache hit - return response
if (response) {
return response;
}
var fetchRequest = event.request.clone();
return fetch(fetchRequest).then(
function(response) {
if(!response || response.status !== 200 || response.type !== 'basic') {
return response;
}
var responseToCache = response.clone();
caches.open(CACHE_NAME)
.then(function(cache) {
cache.put(event.request, responseToCache);
});
return response;
}
);
})
);
});
in case anyone has the same problem,
here is what i should change
self.addEventListener('fetch', function(e) {
console.log('[ServiceWorker] Fetch', e.request.url);
e.respondWith(
caches.match(e.request).then(function(response) {
return response || fetch(e.request);
})
);
});
so my previous fetch looks like the problem
The data on the webpage is displayed dynamically and it seems that checking for every change in the html and extracting the data is a very daunting task and also needs me to use very unreliable XPaths. So I would want to be able to extract the data from the XHR packets.
I hope to be able to extract information from XHR packets as well as generate 'XHR' packets to be sent to the server.
The extracting information part is more important for me because the sending of information can be handled easily by automatically triggering html elements using casperjs.
I'm attaching a screenshot of what I mean.
The text in the response tab is the data I need to process afterwards. (This XHR response has been received from the server.)
This is not easily possible, because the resource.received event handler only provides meta data like url, headers or status, but not the actual data. The underlying phantomjs event handler acts the same way.
Stateless AJAX Request
If the ajax call is stateless, you may repeat the request
casper.on("resource.received", function(resource){
// somehow identify this request, here: if it contains ".json"
// it also also only does something when the stage is "end" otherwise this would be executed two times
if (resource.url.indexOf(".json") != -1 && resource.stage == "end") {
var data = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, resource.url);
// do something with data, you might need to JSON.parse(data)
}
});
casper.start(url); // your script
You may want to add the event listener to resource.requested. That way you don't need to way for the call to complete.
You can also do this right inside of the control flow like this (source: A: CasperJS waitForResource: how to get the resource i've waited for):
casper.start(url);
var res, resData;
casper.waitForResource(function check(resource){
res = resource;
return resource.url.indexOf(".json") != -1;
}, function then(){
resData = casper.evaluate(function(url){
// synchronous GET request
return __utils__.sendAJAX(url, "GET");
}, res.url);
// do something with the data here or in a later step
});
casper.run();
Stateful AJAX Request
If it is not stateless, you would need to replace the implementation of XMLHttpRequest. You will need to inject your own implementation of the onreadystatechange handler, collect the information in the page window object and later collect it in another evaluate call.
You may want to look at the XHR faker in sinon.js or use the following complete proxy for XMLHttpRequest (I modeled it after method 3 from How can I create a XMLHttpRequest wrapper/proxy?):
function replaceXHR(){
(function(window, debug){
function args(a){
var s = "";
for(var i = 0; i < a.length; i++) {
s += "\t\n[" + i + "] => " + a[i];
}
return s;
}
var _XMLHttpRequest = window.XMLHttpRequest;
window.XMLHttpRequest = function() {
this.xhr = new _XMLHttpRequest();
}
// proxy ALL methods/properties
var methods = [
"open",
"abort",
"setRequestHeader",
"send",
"addEventListener",
"removeEventListener",
"getResponseHeader",
"getAllResponseHeaders",
"dispatchEvent",
"overrideMimeType"
];
methods.forEach(function(method){
window.XMLHttpRequest.prototype[method] = function() {
if (debug) console.log("ARGUMENTS", method, args(arguments));
if (method == "open") {
this._url = arguments[1];
}
return this.xhr[method].apply(this.xhr, arguments);
}
});
// proxy change event handler
Object.defineProperty(window.XMLHttpRequest.prototype, "onreadystatechange", {
get: function(){
// this will probably never called
return this.xhr.onreadystatechange;
},
set: function(onreadystatechange){
var that = this.xhr;
var realThis = this;
that.onreadystatechange = function(){
// request is fully loaded
if (that.readyState == 4) {
if (debug) console.log("RESPONSE RECEIVED:", typeof that.responseText == "string" ? that.responseText.length : "none");
// there is a response and filter execution based on url
if (that.responseText && realThis._url.indexOf("whatever") != -1) {
window.myAwesomeResponse = that.responseText;
}
}
onreadystatechange.call(that);
};
}
});
var otherscalars = [
"onabort",
"onerror",
"onload",
"onloadstart",
"onloadend",
"onprogress",
"readyState",
"responseText",
"responseType",
"responseXML",
"status",
"statusText",
"upload",
"withCredentials",
"DONE",
"UNSENT",
"HEADERS_RECEIVED",
"LOADING",
"OPENED"
];
otherscalars.forEach(function(scalar){
Object.defineProperty(window.XMLHttpRequest.prototype, scalar, {
get: function(){
return this.xhr[scalar];
},
set: function(obj){
this.xhr[scalar] = obj;
}
});
});
})(window, false);
}
If you want to capture the AJAX calls from the very beginning, you need to add this to one of the first event handlers
casper.on("page.initialized", function(resource){
this.evaluate(replaceXHR);
});
or evaluate(replaceXHR) when you need it.
The control flow would look like this:
function replaceXHR(){ /* from above*/ }
casper.start(yourUrl, function(){
this.evaluate(replaceXHR);
});
function getAwesomeResponse(){
return this.evaluate(function(){
return window.myAwesomeResponse;
});
}
// stops waiting if window.myAwesomeResponse is something that evaluates to true
casper.waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
});
casper.run();
As described above, I create a proxy for XMLHttpRequest so that every time it is used on the page, I can do something with it. The page that you scrape uses the xhr.onreadystatechange callback to receive data. The proxying is done by defining a specific setter function which writes the received data to window.myAwesomeResponse in the page context. The only thing you need to do is retrieving this text.
JSONP Request
Writing a proxy for JSONP is even easier, if you know the prefix (the function to call with the loaded JSON e.g. insert({"data":["Some", "JSON", "here"],"id":"asdasda")). You can overwrite insert in the page context
after the page is loaded
casper.start(url).then(function(){
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
or before the request is received (if the function is registered just before the request is invoked)
casper.on("resource.requested", function(resource){
// filter on the correct call
if (resource.url.indexOf(".jsonp") != -1) {
this.evaluate(function(){
var oldInsert = insert;
insert = function(json){
window.myAwesomeResponse = json;
oldInsert.apply(window, arguments);
};
});
}
}).run();
casper.start(url).waitFor(getAwesomeResponse, function then(){
var data = JSON.parse(getAwesomeResponse());
// Do something with data
}).run();
I may be late into the party, but the answer may help someone like me who would fall into this problem later in future.
I had to start with PhantomJS, then moved to CasperJS but finally settled with SlimerJS. Slimer is based on Phantom, is compatible with Casper, and can send you back the response body using the same onResponseReceived method, in "response.body" part.
Reference: https://docs.slimerjs.org/current/api/webpage.html#webpage-onresourcereceived
#Artjom's answer's doesn't work for me in the recent Chrome and CasperJS versions.
Based on #Artjom's answer and based on gilly3's answer on how to replace XMLHttpRequest, I have composed a new solution that should work in most/all versions of the different browsers. Works for me.
SlimerJS cannot work on newer version of FireFox, therefore no good for me.
Here is the the generic code to add a listner to load of XHR (not dependent on CasperJS):
var addXHRListener = function (XHROnStateChange) {
var XHROnLoad = function () {
if (this.readyState == 4) {
XHROnStateChange(this)
}
}
var open_original = XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open = function (method, url, async, unk1, unk2) {
this.requestUrl = url
open_original.apply(this, arguments);
};
var xhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function () {
var xhr = this;
if (xhr.addEventListener) {
xhr.removeEventListener("readystatechange", XHROnLoad);
xhr.addEventListener("readystatechange", XHROnLoad, false);
} else {
function readyStateChange() {
if (handler) {
if (handler.handleEvent) {
handler.handleEvent.apply(xhr, arguments);
} else {
handler.apply(xhr, arguments);
}
}
XHROnLoad.apply(xhr, arguments);
setReadyStateChange();
}
function setReadyStateChange() {
setTimeout(function () {
if (xhr.onreadystatechange != readyStateChange) {
handler = xhr.onreadystatechange;
xhr.onreadystatechange = readyStateChange;
}
}, 1);
}
var handler;
setReadyStateChange();
}
xhrSend.apply(xhr, arguments);
};
}
Here is CasperJS code to emit a custom event on load of XHR:
casper.on("page.initialized", function (resource) {
var emitXHRLoad = function (xhr) {
window.callPhantom({eventName: 'xhr.load', eventData: xhr})
}
this.evaluate(addXHRListener, emitXHRLoad);
});
casper.on('remote.callback', function (data) {
casper.emit(data.eventName, data.eventData)
});
Here is a code to listen to "xhr.load" event and get the XHR response body:
casper.on('xhr.load', function (xhr) {
console.log('xhr load', xhr.requestUrl)
console.log('xhr load', xhr.responseText)
});
Additionally, you can also directly download the content and manipulate it later.
Here is the example of the script I am using to retrieve a JSON and save it locally :
var casper = require('casper').create({
pageSettings: {
webSecurityEnabled: false
}
});
var url = 'https://twitter.com/users/username_available?username=whatever';
casper.start('about:blank', function() {
this.download(url, "hop.json");
});
casper.run(function() {
this.echo('Done.').exit();
});
$rootScope.$on("$locationChangeStart", function(event, next, current) {
var partsOfUrl = next.split('/');
var isLogin = false;
if(partsOfUrl.indexOf("signin") > 0) {
isLogin = true;
}
var myDataPromise = loginService.getData();
myDataPromise.then(function(data) { // this is only run after $http completes
if(!isLogin) {
if(data.logout) {
$location.url("pages/signin");
event.preventDefault();
} else{}
} else {
if(data.logout) {
} else {
}
}
});
console.log(next);
});
This is the code i used to check user authentication and prevent the protected areas. But problem here is if a user try to access protected then immediately browser shows the secure page and then get back to login page instead of redirecting to login page first. I think that's because of user authentication process is done through an Ajax call so the program never holds for the response. What's the wrong here and how should i get rid of it ?
Try with httpInterceptor (from mean.io stack)
btw the server should response with a 401 status
'use strict';
angular.module('mean-factory-interceptor',[])
.factory('httpInterceptor', ['$q','$location',function ($q,$location) {
return {
'response': function(response) {
if (response.status === 401) {
$location.path('/signin');
return $q.reject(response);
}
return response || $q.when(response);
},
'responseError': function(rejection) {
if (rejection.status === 401) {
$location.url('/signin');
return $q.reject(rejection);
}
return $q.reject(rejection);
}
};
}
])
//Http Intercpetor to check auth failures for xhr requests
.config(['$httpProvider',function($httpProvider) {
$httpProvider.interceptors.push('httpInterceptor');
}]);
I'm trying to build a front-end for a metrics tool with Ember. The code that I've written so far has been very much influenced by Eviltrout's emberreddit application
https://github.com/eviltrout/emberreddit
The goal is to have two classes that depend on each other: metrics and filters.
1) Once the application initializes, the filters, which are instances of the Filter-class, are loaded from the server. Once the filters have loaded, they are displayed as checkboxes on the screen. After that, the metrics objects should take the filters as parameters and query the server for data.
2) Once the user changes the checkboxes and thus updates the filter objects, the application should take the filters as parameters again and fetch new metrics data from the server.
My problem is that I don't know how to handle the dependencies between these two sets of objects with asynchronous ajax calls. At it's current state, my application doesn't finish loading the filters when it already starts loading the metrics. Therefore, the filters don't get passed as parameters for the metrics ajax-call.
My question is: What's the best way to do this ember? There surely has to be a way to handle the order of ajax calls. My intuition is that manually adding observers isn't the way to go.
Here are the models of my application:
//FILTER MODELS
var defaultFilters = ['dates', 'devices'];
//set Filter class. The Filter object will be multiplied for each filter.
App.Filter = Ember.Object.extend({
//capitalize first letter to get title
filterTitle: function() {
return this.get('id').charAt(0).toUpperCase() + this.get('id').slice(1);
}.property('id'),
//set attribute to see if filter has loaded
loadedFilter: false,
//create method to load filter values from server
loadValues: function() {
var filter = this;
return Ember.Deferred.promise(function (p) {
if (filter.get('loadedFilter')) {
p.resolve(filter.get('values'));
} else {
p.resolve($.getJSON("http://127.0.0.1:13373/options/" + filter.get('id')).then(function(response) {
var values = Ember.A();
response[filter.get('id')].forEach(function(value) {
values.push(value);
});
filter.setProperties({values: values, loadedFilter: true});
return values;
}))
}})}
}
);
//reopen class to create "all" method which returns all instances of Filter class
App.Filter.reopenClass({
all: function() {
if (this._all) {return this._all; }
var all = Ember.A();
defaultFilters.forEach(function(id) {
all.pushObject(App.Filter.create({id: id}));
});
this._all = all;
return all;
}});
//Create a Filters array to store all the filters.
App.Filters = App.Filter.all();
//METRIC MODELS
App.Metric = Ember.Object.extend({
metricTitle: function() {
return this.get('id').charAt(0).toUpperCase() + this.get('id').slice(1);
}.property('id'),
loadedMetric: false,
filtersBinding: 'App.Filters',
loadValues: function() {
var metric = this;
var filters = metric.get('filters');
if (filters.get('loadedFilters'))
console.log('loading metrics');
return Ember.Deferred.promise(function (p) {
if (metric.get('loadedMetric')) {
p.resolve(metric.get('values'));
} else {
p.resolve(
console.log('sending ajax'),
$.ajax({
url: "http://127.0.0.1:13373/" + metric.get('id') + "/",
data: JSON.stringify(metric.get('filters')),
}).then(function(response) {
var values = Ember.A();
response[metric.get('id')].forEach(function(value) {
values.push(value);
});
metric.setProperties({"values": values, "loadedMetric": true});
return values;
}))
}})}
});
App.Metric.reopenClass({
findByView: function(searchView) {
if (this._metrics) {return this._metrics; }
var metrics = Ember.A();
defaultMetricsSettings.forEach(function(metric) {
if (metric.view == searchView)
metrics.pushObject(App.Metric.create({id: metric.id},{view: metric.view}, {calculation: metric.calculation}, {format: metric.format}, {width: metric.width}));
});
this._metrics = metrics;
return metrics;
}
});
And here are the routes:
App.ApplicationRoute = Ember.Route.extend({
//set application routes model to all filters
model: function() {
return App.Filter.all();
},
//after filter has loaded, let's load its values
afterModel: function(model) {
return model.forEach(function(item) {
item.loadValues();
});
},
//create a controller called ApplicationController and pass the filter as its model
setupController: function(controller, filter) {
controller.set('model', filter);
}
});
App.DashboardRoute = Ember.Route.extend({
model: function() {
return App.Metric.findByView('Dashboard');
},
afterModel: function(model) {
return model.forEach(function(item) {
item.loadValues();
});
},
setupController: function(controller, metric) {
controller.set('model', metric);
}
});
Controllers:
App.ApplicationController = Ember.ArrayController.extend({
//ApplicationController controls all the filters. Let's create a controller to handle each instance of a filter
itemController: 'filter'
});
App.FilterController = Ember.ObjectController.extend({
//this sets the titleId property that is used only for binding html attributes in template. Stupid way to do this.
titleId: function() {
return "#" + this.get('filterTitle');}.property('filterTitle')
});
Your afterModel hook could do this in a sequence of dependent promises. The current implementation is returning immediately, instead you chain the promise and finally return the last promise as the result of the hook. The router will wait for the whole set of calls to complete before continuing to setupController.
afterModel: function(model) {
var promise;
model.forEach(function(item)) {
if (promise) {
promise = promise.then(function() {
item.loadValues();
});
} else {
promise = item.loadValues();
}
}
return promise;
}
I'm not sure how many of the calls you have, but you may want to batch some of these together to reduce the number of HTTP requests.