I have an array with the following data for each node
[{
"id": "2130483",
"appId": "SIGERprod",
"time": 1500,
"dateStart": 1521564131000,
"timestamp": 1521564131000,
"dateEnd": 1521564131000,
"ipAddress": "10.110.11.111",
"principalName": "suiza_2#hotmail.com",
"contextPath": "/prueba",
"ruta": "/prueba/xhtml/formasPreCodificadas/llenarForma.xhtml",
"metodo": "POST",
"status": 200,
"year": 2018,
"month": 3,
"day": 20,
"hour": 10,
"minute": 42,
"second": 11
}]
I have the following code and this code grouped my request per date
$.getJSON(prefix + "/getFullData.htm", {idApp: App, dateStart: inicio, dateEnd: fin},
function (data) {
data.forEach(function (d) {
d.date = new Date(d.year, d.month, d.day, d.hour, d.minute, d.second, 0);
});
ndx = crossfilter(data);
dateDimension = ndx.dimension(function (d) {
return d.date;
});
var peticiones = dateDimension.group().reduceCount();
var minDate = dateDimension.bottom(1)[0].date;
var maxDate = dateDimension.top(1)[0].date;
solicitudesXIntervalo
.width(1000)
.height(200)
.mouseZoomable(true)
.dimension(dateDimension)
.group(peticiones)
.x(d3.time.scale().domain([minDate, maxDate]))
.yAxisLabel("request grouped by second");
and I get the following
I need to graph the time it takes each request (time vs date)
It sounds like you want the requests to be summed based on their durations rather than just counted.
If this is what you mean, it's a simple change:
var peticiones = dateDimension.group().reduceSum(function(d) {
return (d.dateEnd - d.dateStart) / 1000;
})
I am dividing by 1000 to get seconds instead of milliseconds; adjust as needed.
However, in your example data above, the duration is 0 (and wouldn't show up at all). So I am not sure if I understand your question. Please comment and/or edit your question if I've missed the mark.
Related
Most examples I have found are using data that has time and number
var data = [
{
"Time": "19-Jan-2018 11:24:49.000 UTC",
"Speed": 1.885
},
{
"Time": "19-Jan-2018 11:24:59.000 UTC",
"Speed": 1.875
},
{
"Time": "19-Jan-2018 11:25:00.000 UTC",
"Speed": 1.878
},
{
"Time": "19-Jan-2018 11:25:01.000 UTC",
"Speed": 1.876
}
]
I am looking to stack type
var data = [
{
"Time": "19-Jan-2018 11:24:49.000 UTC",
"type": "CAT"
},
{
"Time": "19-Jan-2018 11:24:59.000 UTC",
"type": "DOG"
},
{
"Time": "19-Jan-2018 11:25:00.000 UTC",
"type": "CAT"
},
{
"Time": "19-Jan-2018 11:25:01.000 UTC",
"Type": "BAT"
}
]
How can I stack categorical data, while allowing the user to select time/category pairs, as in the following Example?
I adapted the example to time/category data in this fiddle.
Those dates would only parse in Chrome, so
const parseDate = d3.utcParse("%d-%b-%Y %H:%M:%S.%L UTC");
data.forEach(d => {
d.Time = parseDate(d.Time);
})
I changed the key functions to use ,
function multikey(x,y) {
return x + ',' + y;
}
function splitkey(k) {
return k.split(',');
}
I also changed fake group stack_second to convert string-dates from the multikeys back into Dates, and to initialize categories to 0 (since every stack has to be present for every X).
function stack_second(group, categories) {
return {
all: function() {
var all = group.all(),
m = {};
// build matrix from multikey/value pairs
all.forEach(function(kv) {
var ks = splitkey(kv.key);
m[ks[0]] = m[ks[0]] || Object.fromEntries(categories.map(c=>[c,0]));
m[ks[0]][ks[1]] = kv.value;
});
// then produce multivalue key/value pairs
return Object.keys(m).map(function(k) {
return {key: new Date(k), value: m[k]};
});
}
};
}
Get the array of categories from the source data:
const categories = Array.from(new Set(data.map(d => d.Type)).values());
When dealing with date/time, you have to choose a d3 time interval appropriate for your data. Here minutes looked right. Using UTC d3-time methods everywhere because your source data is UTC.
const interval = d3.utcMinute;
Calculate xscale domain and apply:
let extent = d3.extent(data, d=>d.Time);
extent[1] = interval.offset(extent[1], 1)
chart
.x(d3.scaleTime().domain(extent))
.xUnits(interval.range)
Right number of ticks, also formatted in UTC:
chart.xAxis().ticks(d3.utcMinute).tickFormat(d3.utcFormat('%H:%M'))
Match colors between stacks and wedges with
.colors(d3.scaleOrdinal().domain(categories).range(d3.schemeCategory10))
Crossfilter initialization, using the interval and categories:
const interval = d3.utcMinute; // choose appropriate to your data
var cf = crossfilter(data),
timeTypeDim = cf.dimension(function(d) { return multikey(interval(d.Time), d.Type); }),
timeTypeGroup = timeTypeDim.group(), // reduceCount by default
stackedGroup = stack_second(timeTypeGroup, categories);
And here's the chart code for completeness, although we've already discussed the relevant parts:
function sel_stack(i) {
return function(d) {
return d.value[i];
};
}
chart
.width(600)
.height(400)
.colors(d3.scaleOrdinal().domain(categories).range(d3.schemeCategory10))
.controlsUseVisibility(true)
.x(d3.scaleTime().domain(extent))
.xUnits(interval.range)
.margins({left: 80, top: 20, right: 10, bottom: 20})
.brushOn(false)
.clipPadding(10)
.title(function(d) {
return d.key + '[' + this.layer + ']: ' + d.value[this.layer];
})
.legend(dc.legend().x(540).y(50))
.dimension(timeTypeDim)
.group(stackedGroup, categories[0], sel_stack(categories[0]))
.renderLabel(true);
Let's say I have a 100 years worth of monthly data, total of 1200 data points, see bottom.
To plot a tiny overview line chart (e.g. just 100 data points) I have to do it manually by grouping. For instance, group the data by year, then get the average of 12 months value, iterate through every group, then finally reduced the data points to 100.
Instead of this approach, is there a convenient way using crossfilter or any other library?
[
{ date: 1900-01, value: 72000000000},
{ date: 1900-02, value: 58000000000},
{ date: 1900-03, value: 2900000000},
{ date: 1900-04, value: 31000000000},
{ date: 1900-05, value: 33000000000},
...
{ date: 1999-11, value: 30000000000},
{ date: 1999-12, value: 10000000000},
]
It's going to be the same algorithm no matter which library you use, just different ways of specifying it. In this case d3.nest is probably the easiest way to do this, but if you want quick filtering, the crossfilter way isn't too bad.
The difference between using d3.nest and crossfilter is that we're not constructing an array of values, just a single value. So we'll maintain both sum and count.
We'll also need to specify what happens when a row is removed from a bin.
var parse = d3.timeParse("%Y-%m");
data.forEach(function(d) {
// it's best to convert fields before passing to crossfilter
// because crossfilter will look at them many times
d.date = parse(d.key);
});
var cf = crossfilter(data);
var yearDim = cf.dimension(d => d3.timeYear(d.date));
var yearAvgGroup = yearDim.group().reduce(
function(p, v) { // add
p.sum += v.value;
++p.count;
p.avg = p.sum/p.count;
return p;
},
function(p, v) { // remove
p.sum -= v.value;
--p.count;
p.avg = p.count ? p.sum/p.count : 0;
return p;
},
function() { // init
return {sum: 0, count: 0, avg: 0};
}
);
Now yearAvgGroup.all() will return an array of key/value pairs, where the key is the year, and the value contains sum, count, and avg.
Crossfilter doesn't make this problem particularly convenient to solve, but reductio has a helper function for this:
var yearAvgGroup = yearDim.group();
reductio().avg(d => d.value);
Note: it doesn't matter unless you have ton of data, but it's more efficient to only compute sum and count in the group, and compute the average when it's needed.
If you're using dc.js, you can use valueAccessor for this:
// remove avg lines from the above, and
chart.dimension(yearDim)
.group(yearAvgGroup)
.valueAccessor(kv => kv.value.sum / kv.value.count);
Assuming your question is only concerned with producing the data, you can use d3-nest, without crossfilter, to average each year:
Parsing the date value, you can then format the date as a year to create a key. This groups values by key, then we rollup those values with a function to calculate the mean for a given year:
var parse = d3.timeParse("%Y-%m"); // takes: "1900-01"
var format = d3.timeFormat("%Y"); // gives: "1900"
var means = d3.nest()
.key(function(d) { return format(parse(d.date)); })
.rollup(function(values) { return d3.mean(values, function(d) {return d.value; }) })
.entries(data);
Which gives us the following structure:
[
{
"key": "1900",
"value": 39380000000
},
{
"key": "1999",
"value": 20000000000
}
]
var data = [
{ date: "1900-01", value: 72000000000},
{ date: "1900-02", value: 58000000000},
{ date: "1900-03", value: 2900000000},
{ date: "1900-04", value: 31000000000},
{ date: "1900-05", value: 33000000000},
{ date: "1999-11", value: 30000000000},
{ date: "1999-12", value: 10000000000},
];
var parse = d3.timeParse("%Y-%m");
var format = d3.timeFormat("%Y");
var means = d3.nest()
.key(function(d) { return format(parse(d.date)); })
.rollup(function(values) { return d3.mean(values, function(d) {return d.value; }) })
.entries(data);
console.log(means);
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>
I would like to show sum of multiple values as one chart output with amCharts. I am using dataLoader with JSON to get the data. I know I have to create a function for but I couldn't understand how to get the data from the dataLoader to calculate
{
"balloonText": "[[title]] of [[valueAxis]]:[[value]]",
"lineThickness": 3,
"id": "sumValue",
"title": "sum Value",
"valueField": (function() {
var sumValues = "calculation";
return sumValues
}
this attempt is probably not correct but this is how I started
{
"balloonText": "[[title]] of [[valueAxis]]:[[value]]",
"lineThickness": 3,
"id": "LoadigTime",
"title": "Loadig Time",
"valueField": (function() {
var sumValues = (HomePageLoad + LoginToParametersLoad + ParametersLoad + AlarmsLoad + SwitchSideLoad + LoginToAdminLoad + AdminLoad) / 7;
return sumValues
})
}
valueField cannot be a function, only a string reference to a field in your data.
If the chart is meant to be displaying the sum of all of those fields in your data as a chart, simply add logic to your postProcess callback to create a new dataset containing your sums, e.g.
postProcess: function(data) {
var newData = [];
data.forEach(function(dataItem) {
var item = {
YOUR_CATEGORY_FIELD: dataItem.YOUR_CATEGORY_FIELD, //replace with your category field name
sum: 0
};
//loop through your item's keys and sum everything up, filtering out
//your category property
item.sum = Object.keys(dataItem).reduce(function(sum, key) {
if (key !== "YOUR_CATEGORY_FIELD") {
sum += dataItem[key]
}
return sum;
}, 0);
newData.push(item);
});
return newData;
},
// ...
graphs: [{
valueField: "sum",
// other props here
}]
In a previous version of my code I used to set the appropriate locale format like this
format = {
"decimal": ".",
"thousands": "",
"grouping": [3],
"currency": ["€", ""],
"dateTime": "%a %b %e %X %Y",
"date": "%d-%m-%Y",
"time": "%H:%M:%S",
"periods": ["AM", "PM"],
"days": ["Domenica", "Lunedi", "Martedi", "Mercoledi", "Giovedi", "Venerdi", "Sabato"],
"shortDays": ["Do", "Lu", "Ma", "Me", "Gi", "Ve", "Sa"],
"months": ["Gennaio", "Febbraio", "Marzo", "Aprile", "Maggio", "Giugno", "Luglio", "Agosto", "Settembre", "Ottobre", "Novembre", "Dicembre"],
"shortMonths": ["Gen", "Feb", "Mar", "Apr", "Mag", "Giu", "Lug", "Ago", "Set", "Ott", "Nov", "Dic"]
}
and then
var localeFormatter = d3.locale(format);
// set time tick format
var tickFormat = localeFormatter.timeFormat.multi([
["%H:%M", function (d) { return d.getMinutes(); }],
["%H:%M", function (d) { return d.getHours(); }],
["%a %d", function (d) { return d.getDay() && d.getDate() != 1; }],
["%b %d", function (d) { return d.getDate() != 1; }],
["%B", function (d) { return d.getMonth(); }],
["%Y", function () { return true; }]
]);
I finally stored these tick format settings so I can use them in my charts
D3Preferences['localTimeTickFormat'] = tickFormat;
After updating to release v4.2.8 d3.locale is gone and I cannot figure out how to achieve the same result.
Can someone point me in the right direction? The d3 documentation did not help me
With .multi deprecated, your tickFormat() function now has to handle filtering logic as well, like this:
// Establish the desired formatting options using locale.format():
var formatDay = d3.timeFormat("%a %d"),
formatWeek = d3.timeFormat("%b %d"),
formatMonth = d3.timeFormat("%B"),
formatYear = d3.timeFormat("%Y");
// Define filter conditions
function tickFormat(date) {
return (d3.timeMonth(date) < date ? (d3.timeWeek(date) < date ? formatDay : formatWeek)
: d3.timeYear(date) < date ? formatMonth
: formatYear)(date);
}
Here's an updated version of Mike's original bl.ock (the one from which you likely derived localeFormatter.timeFormat.multi()), set to use the conditional logic #altocumulus mentions above.
Simplely you can do it like this
d3.formatDefaultLocale(format);
and then
var tickFormat=function(date){
if(date.getMinutes()) return d3.timeFormat('%H:%M')(date);
if(date.getHours()) return d3.timeFormat('%H:%M')(date);
if(date.getDay()&&date.getDate()!=1) return d3.timeFormat('%a %d')(date);
if(date.getDate()!=1) return d3.timeFormat('%b %d')(date);
if(date.getMonth()) return d3.timeFormat('%B')(date);
return d3.timeFormat('%Y')(date);
}
It works for me.
Here is the jfiddle - http://jsfiddle.net/inasisi/6v639g9g/1/
As you can see the X axis is not scaled properly. I can calculate the min and max date and set the scale properly but don't want to do it after each filter. Would prefer if elasticX works properly.
Any ideas?
var chartGroup = "chartGroup";
data = [{
"run_date": "2013-01-20",
"current_grade": "Kindergarten",
"students": 1
}, {
"run_date": "2013-01-20",
"current_grade": "First",
"students": 2
}, {
"run_date": "2014-03-22",
"current_grade": "Kindergarten",
"students": 3
}, {
"run_date": "2014-03-22",
"current_grade": "First",
"students": 4
}, {
"run_date": "2015-10-06",
"current_grade": "Kindergarten",
"students": 5
}, {
"run_date": "2015-10-06",
"current_grade": "First",
"students": 21
}, {
"run_date": "2015-02-13",
"current_grade": "Kindergarten",
"students": 31
}, {
"run_date": "2015-02-13",
"current_grade": "First",
"students": 26
}, ];
var ndx = crossfilter(data);
var dateFormat = d3.time.format("%Y-%m-%d");
data.forEach(function (d) {
d.run_date = Date.parse(d.run_date);
});
var ndx = crossfilter(data);
filterDateDimension = ndx.dimension(function (d) {
return [d.run_date];
});
dateDimension = ndx.dimension(function (d) {
return [d.run_date];
});
var minDate = dateDimension.bottom(1)[0].run_date;
var maxDate = dateDimension.top(1)[0].run_date;
var runsStudentsGroup = dateDimension.group().reduceSum(function (fact) {
return fact.students;
});
var totalStudentsChart = dc.lineChart("#students_chart", chartGroup);
totalStudentsChart.renderArea(true)
.width(300)
.height(300)
.x(d3.time.scale())
.elasticY(true)
.elasticX(true)
.renderHorizontalGridLines(true)
.renderVerticalGridLines(true)
.dimension(dateDimension)
//.colors('red')
.group(runsStudentsGroup);
dc.renderAll(chartGroup);
$('.day_filter').on('click', function () {
console.log(dateDimension.top(Infinity));
console.log($(this).val());
dateDimension.filter(function (d) {
console.log(d > new Date(2015, 0, 1));
return d > new Date(2015, 0, 1);
});
console.log(dateDimension.top(Infinity));
dc.redrawAll();
});
I had to fix a few things to get the chart to display and to get the filter to work at all. I'll just quote those without explaining, since those aren't what the question is about:
d.run_date = new Date(d.run_date);
//...
return d.run_date; // twice
//...
filterDateDimension.filter(function (d) {
//...
dc.redrawAll(chartGroup);
To answer your main question, which is frequently asked, crossfilter does not automatically remove empty bins. You can use a "fake group" to filter them out.
Adding:
function remove_empty_bins(source_group) {
return {
all:function () {
return source_group.all().filter(function(d) {
return d.value != 0;
});
}
};
}
//...
.group(remove_empty_bins(runsStudentsGroup));
Working fork of your fiddle here: http://jsfiddle.net/gordonwoodhull/8an2n1eL/5/
(The transition in this example is particularly screwy, and will be fixed in 2.1.)