I have an array with objects (key, values):
var array = [ {
key: 'Monday',
values: [
['time', 123], ['someUnixTime', 1234] ...
]
} ,
.... // here more same objects
];
I don't know how to handle groups and dimensions, I mean, dimension should return an exact value, but what about in case I have an array of arrays to return?
var ndx = crossfilter(array);
var dimensionByTime = ndx.dimension(function (d) {
return d.values; // I need here first element of array
});
var dimensionValueByTime = dimension.group().reduceSum(function (d) {
return d.key; // I need here 2nd element of array
});
Thanks in advance!
Related
Let's say I have a 100 years worth of monthly data, total of 1200 data points, see bottom.
To plot a tiny overview line chart (e.g. just 100 data points) I have to do it manually by grouping. For instance, group the data by year, then get the average of 12 months value, iterate through every group, then finally reduced the data points to 100.
Instead of this approach, is there a convenient way using crossfilter or any other library?
[
{ date: 1900-01, value: 72000000000},
{ date: 1900-02, value: 58000000000},
{ date: 1900-03, value: 2900000000},
{ date: 1900-04, value: 31000000000},
{ date: 1900-05, value: 33000000000},
...
{ date: 1999-11, value: 30000000000},
{ date: 1999-12, value: 10000000000},
]
It's going to be the same algorithm no matter which library you use, just different ways of specifying it. In this case d3.nest is probably the easiest way to do this, but if you want quick filtering, the crossfilter way isn't too bad.
The difference between using d3.nest and crossfilter is that we're not constructing an array of values, just a single value. So we'll maintain both sum and count.
We'll also need to specify what happens when a row is removed from a bin.
var parse = d3.timeParse("%Y-%m");
data.forEach(function(d) {
// it's best to convert fields before passing to crossfilter
// because crossfilter will look at them many times
d.date = parse(d.key);
});
var cf = crossfilter(data);
var yearDim = cf.dimension(d => d3.timeYear(d.date));
var yearAvgGroup = yearDim.group().reduce(
function(p, v) { // add
p.sum += v.value;
++p.count;
p.avg = p.sum/p.count;
return p;
},
function(p, v) { // remove
p.sum -= v.value;
--p.count;
p.avg = p.count ? p.sum/p.count : 0;
return p;
},
function() { // init
return {sum: 0, count: 0, avg: 0};
}
);
Now yearAvgGroup.all() will return an array of key/value pairs, where the key is the year, and the value contains sum, count, and avg.
Crossfilter doesn't make this problem particularly convenient to solve, but reductio has a helper function for this:
var yearAvgGroup = yearDim.group();
reductio().avg(d => d.value);
Note: it doesn't matter unless you have ton of data, but it's more efficient to only compute sum and count in the group, and compute the average when it's needed.
If you're using dc.js, you can use valueAccessor for this:
// remove avg lines from the above, and
chart.dimension(yearDim)
.group(yearAvgGroup)
.valueAccessor(kv => kv.value.sum / kv.value.count);
Assuming your question is only concerned with producing the data, you can use d3-nest, without crossfilter, to average each year:
Parsing the date value, you can then format the date as a year to create a key. This groups values by key, then we rollup those values with a function to calculate the mean for a given year:
var parse = d3.timeParse("%Y-%m"); // takes: "1900-01"
var format = d3.timeFormat("%Y"); // gives: "1900"
var means = d3.nest()
.key(function(d) { return format(parse(d.date)); })
.rollup(function(values) { return d3.mean(values, function(d) {return d.value; }) })
.entries(data);
Which gives us the following structure:
[
{
"key": "1900",
"value": 39380000000
},
{
"key": "1999",
"value": 20000000000
}
]
var data = [
{ date: "1900-01", value: 72000000000},
{ date: "1900-02", value: 58000000000},
{ date: "1900-03", value: 2900000000},
{ date: "1900-04", value: 31000000000},
{ date: "1900-05", value: 33000000000},
{ date: "1999-11", value: 30000000000},
{ date: "1999-12", value: 10000000000},
];
var parse = d3.timeParse("%Y-%m");
var format = d3.timeFormat("%Y");
var means = d3.nest()
.key(function(d) { return format(parse(d.date)); })
.rollup(function(values) { return d3.mean(values, function(d) {return d.value; }) })
.entries(data);
console.log(means);
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>
Can not find Rx.Observable.pairs in Rxjs5,
what I need just convert an object into Observable and inspect the change for each property.
any ideas?
var a = { aa: "aa", bb: "bb" };
function pairs(obj) {
// List of object's key-value pairs
var keyValuePairs = Object.keys(obj).map(key => ({ key, value: obj[key] }));
// Convert to an Observable and return
return Rx.Observable.from(keyValuePairs);
}
var xxx = pairs(a);
xxx.subscribe(x => {
console.log(x);
})
a.aa = "mm";
You can accomplish this from scratch:
function pairs(obj) {
// List of object's key-value pairs
var keyValuePairs = Object.keys(obj).map(key => ({ key, value: obj[key]}));
// Convert to an Observable and return
return Rx.Observable.from(keyValuePairs);
}
I have data that looks like this:
var records = [
{id: '1', cat: 'A'},
{id: '2', cat: 'A'},
{id: '3', cat: 'B'},
{id: '4', cat: 'B'},
{id: '5', cat: 'B'},
{id: '6', cat: 'C'}
];
I want to create a dc.numberDisplay that displays the count of the number of unique categories, 3 in the example data above (A, B, & C).
This is what I'm currently doing:
var ndx = crossfilter(data); // init crossfilter
// create dimension based on category
var categoryDimension = ndx.dimension(
function (d) {
return d.category;
}
);
// Group by category
var categoryGroup = categoryDimension.group();
var categoryCount = dc.numberDisplay('#category-count'); // An empty span
categoryCount
.group(categoryGroup)
.valueAccessor(
function (d) { return d.value; }
);
The problem is that the numberDisplay displays 2 instead of 3. When debugging, I found that when the valueAccessor is called, d is the count of the number of elements of category A instead of the count of the number of categories.
How can I solve this problem?
UPDATE: Thanks to Nathan's solution, here is a working code snippet (ES2016 style)
const categoryDimension = claims.dimension(
(d) => {
return d.cat;
}
);
const categoryGroup = categoryDimension.groupAll().reduce(
(p, v) => { // add element
const cat = v.cat;
const count = p.categories.get(cat) || 0;
p.categories.set(cat, count + 1);
return p;
},
(p, v) => { // remove element
const cat = v.cat;
const count = p.categories.get(cat);
if (count === 1) {
p.categories.delete(cat);
} else {
p.categories.set(cat, count - 1);
}
return p;
},
() => { // init
return {
categories: new Map()
};
});
categoryCount
.group(categoryGroup)
.valueAccessor(
(d) => {
return d.categories.size;
}
);
You will need to use groupAll() since the number-display only looks at the top group. Then provide custom reduce functions to track unique categories. Finally, when DC.js pulls the value from the top group (there is only one) - just return the number of categories (which is the number of keys in the p object).
var categoryGroup = categoryDimension.groupAll().reduce(
function (p, v) { //add
if(p[v.cat]) {
p[v.cat]++;
} else {
p[v.cat] = 1;
}
return p;
},
function (p, v) { //remove
p[v.cat]--;
if(p[v.cat] === 0) {
delete p[v.cat];
}
return p;
},
function () { //init
//initial p - only one since using groupAll
return {};
}
);
console.debug("groups", categoryGroup.value());
dc.numberDisplay('#category-count')
.group(categoryGroup)
.valueAccessor(
function (d) { return Object.keys(d).length; }
);
I am looking for how to create a rowchart in dc.js to show and filter items with multiple tags. I've summed up a few answers given on stack overflow, and now have a working code.
var data = [
{id:1, tags: [1,2,3]},
{id:2, tags: [3]},
{id:3, tags: [1]},
{id:4, tags: [2,3]},
{id:5, tags: [3]},
{id:6, tags: [1,2,3]},
{id:7, tags: [1,2]}];
var content=crossfilter(data);
var idDimension = content.dimension(function (d) { return d.id; });
var grid = dc.dataTable("#idgrid");
grid
.dimension(idDimension)
.group(function(d){ return "ITEMS" })
.columns([
function(d){return d.id+" : "; },
function(d){return d.tags;},
])
function reduceAdd(p, v) {
v.tags.forEach (function(val, idx) {
p[val] = (p[val] || 0) + 1; //increment counts
});
return p;
}
function reduceRemove(p, v) {
v.tags.forEach (function(val, idx) {
p[val] = (p[val] || 0) - 1; //decrement counts
});
return p;
}
function reduceInitial() {
return {};
}
var tags = content.dimension(function (d) { return d.tags });
var groupall = tags.groupAll();
var tagsGroup = groupall.reduce(reduceAdd, reduceRemove, reduceInitial).value();
tagsGroup.all = function() {
var newObject = [];
for (var key in this) {
if (this.hasOwnProperty(key) && key != "") {
newObject.push({
key: key,
value: this[key]
});
}
}
return newObject;
}
var tagsChart = dc.rowChart("#idtags")
tagsChart
.width(400)
.height(200)
.renderLabel(true)
.labelOffsetY(10)
.gap(2)
.group(tagsGroup)
.dimension(tags)
.elasticX(true)
.transitionDuration(1000)
.colors(d3.scale.category10())
.label(function (d) { return d.key })
.filterHandler (function (dimension, filters) {
var fm = filters.map(Number)
dimension.filter(null);
if (fm.length === 0)
dimension.filter(null);
else
dimension.filterFunction(function (d) {
for (var i=0; i < fm.length; i++) {
if (d.indexOf(fm[i]) <0) return false;
}
return true;
});
return filters;
}
)
.xAxis().ticks(5);
It can be seen on http://jsfiddle.net/ewm76uru/24/
Nevertheless, the rowchart is not updated when I filter by one tag. For example, on jsfiddle, if you select tag '1', it filters items 1,3,6 and 7. Fine. But the rowchart is not updated... I Should have tag '3' count lowered to 2 for example.
Is there a way to have the rowchart tags counts updated each time I filter by tags ?
Thanks.
After a long struggle, I think I have finally gathered a working solution.
As said on crossfilter documentation : "a grouping intersects the crossfilter's current filters, except for the associated dimension's filter"
So, the tags dimension is not filtered when tag selection is modified, and there is no flag or function to force this reset. Nevertheless, there is a workaround (given here : https://github.com/square/crossfilter/issues/146).
The idea is to duplicate the 'tags' dimension, and to use it as the filtered dimension :
var tags = content.dimension(function (d) { return d.tags });
// duplicate the dimension
var tags2 = content.dimension(function (d) { return d.tags });
var groupall = tags.groupAll();
...
tagsChart
.group(tagsGroup)
.dimension(tags2) // and use this duplicated dimension
as it can been seen here :
http://jsfiddle.net/ewm76uru/30/
I hope this will help.
I have the following data structure and would like to display a lineChart with the minimum value of 'amount' (group) by Day (of date) (dimension).
var data = [
{date: "2014-01-01", amount: 10},
{date: "2014-01-01", amount: 1},
{date: "2014-01-15", amount: 0},
{date: "2014-01-15", amount: 10 },
{date: "2014-02-20", amount: 100 },
{date: "2014-02-20", amount: 10 },
];
Where as I would normally be doing something along the following lines, I'm not sure how to find the min in the group.
var dateDim = facts.dimension(function (d) {return d3.time.day(d.date);});
var dateDimGroup = dateDim.group().reduceSum(function (d) { return d.amount; })
Is this possible? I can't seem to find any examples of this so any help would be really appreciated.
Thanks!
You're going to need to keep a custom grouping. The basic idea is that you maintain an array (ordered is best) of all the values in a group. In your add function, you the current value to the array and assign the first value to a 'min' property of the group. In your remove function you remove values, you remove the current value from the array and then assign the first value to the 'min' property (and check if the list is empty, in which case set it to undefined or something along those lines).
You functions will look something like this:
function add(accessor) {
var i;
var bisect = crossfilter.bisect.by(function(d) { return d; }).left;
return function (p, v) {
// Not sure if this is more efficient than sorting.
i = bisect(p.valueList, accessor(v), 0, p.valueList.length);
p.valueList.splice(i, 0, accessor(v));
p.min = p.valueList[0];
return p;
};
};
function remove(accessor) {
var i;
var bisect = crossfilter.bisect.by(function(d) { return d; }).left;
return function (p, v) {
i = bisect(p.valueList, accessor(v), 0, p.valueList.length);
// Value already exists or something has gone terribly wrong.
p.valueList.splice(i, 1);
p.min = p.valueList[0];
return p;
};
};
function initial() {
return function () {
return {
valueList: [],
min: undefined
};
};
}
'accessor' is a function to get at the value you want the minimum of, like in reduceSum. Call each of these functions to return the function you use in the corresponding place in the .group(add, remove, initial) function.
This code is pretty much ripped straight out of reductio, so you could also use reductio and do:
var dateDim = facts.dimension(function (d) {return d3.time.day(d.date);});
var dateDimGroup = reductio().min(function (d) { return d.amount; })(dateDim.group());
Your dateDimGroup will then have a 'min' property that should give you the minimum amount for every date.