D3 cross tabulation HTML table - d3.js

I'm trying to create a D3 cross tabulation HTML table (there will be more interactive features, this is just the first step) based on JSON data. I can populate the horizontal table header, but am having trouble with the vertical headers and data fields.
The table should look something like the following:
My code so far (JSFiddle here) is:
var nested = d3.nest()
.key(function(d) { return d._id.env; })
.entries(data.result);
console.log(nested);
var table = d3.select("#table")
.append("table")
.style("border-collapse", "collapse")
.style("border", "2px black solid");
var thead = table.append("thead");
thead.append("tr")
.selectAll("th")
.data(nested)
.enter().append("th")
.text(function(d) { return d.key; });
var tbody = table.append("tbody");
var tr = tbody.selectAll("tr")
.data(nested.values) // not sure how to get this
.enter().append("tr");
tr.selectAll("td")
.data(function(d) { console.log(d); return d; })
.enter().append("td")
.text(function(d, i) { console.log(d[i]); return d; });
The raw data is in the following format:
{
"result" : [
{
"_id" : {
"month" : 5,
"day" : 6,
"year" : 2014,
"env" : "A"
},
"ruleScore" : 83.25,
"jsPerPage" : 12,
"cssPerPage" : 4,
"imagesPerPage" : 7.75
},
{
"_id" : {
"month" : 5,
"day" : 6,
"year" : 2014,
"env" : "B"
},
"ruleScore" : 83,
"jsPerPage" : 12,
"cssPerPage" : 4,
"imagesPerPage" : 10
},
{
"_id" : {
"month" : 5,
"day" : 6,
"year" : 2014,
"env" : "C"
},
"ruleScore" : 83,
"jsPerPage" : 12,
"cssPerPage" : 5,
"imagesPerPage" : 10,
},
{
"_id" : {
"month" : 5,
"day" : 6,
"year" : 2014,
"env" : "D"
},
"ruleScore" : 83.08333333333333,
"jsPerPage" : 12,
"cssPerPage" : 6,
"imagesPerPage" : 9.25
}
],
"ok" : 1
}
I'm guessing this is pretty simple, just can't get my head around it - thanks!

Related

Nest - Need help on elastic response mocking

"aggregations" : {
"filter#count_stats" : {
"doc_count" : 30,
"lterms#Name1" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 53986,
"doc_count" : 2,
"sterms#Name2" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "Soft",
"doc_count" : 7,
},
{
"key" : "Health",
"doc_count" : 5
},
]
}
},
{
"key" : 40127,
"doc_count" : 1,
"sterms#Name3" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "XYZ",
"doc_count" : 3
}
]
}
}
]
}
}
}
IReadOnlyDictionary<string, IAggregate> dictionary = new Dictionary<string, IAggregate>();
var keyedAggregate1 = new KeyedBucket<object>(dictionary) { Key = "Soft", DocCount = 7};
var keyedAggregate2 = new KeyedBucket<object>(dictionary) { Key = "Health", DocCount = 8 };
var keyedAggregate3 = new KeyedBucket<object>(dictionary) { Key = "XYZ", DocCount = 3 };
var backingListDeals1 = new List<IBucket>
{
keyedAggregate1,
keyedAggregate2
};
var backingListDeals2 = new List<IBucket>
{
keyedAggregate3
};
var bucketAggregate1 = new BucketAggregate()
{
Items = backingListDeals1,
DocCount = 2
};
var bucketAggregate2 = new BucketAggregate()
{
Items = backingListDeals2,
DocCount = 2
};
var backingDictionary = new Dictionary<string, IAggregate> {{"count_stats", new BucketAggregate
{
Items = new List<IBucket>
{
new KeyedBucket<object>(new Dictionary<string, IAggregate>{{ "Name2", bucketAggregate1 } })
{
Key = "53986",
DocCount = 2
},
new KeyedBucket<object>(new Dictionary<string, IAggregate>{{ "Name2", bucketAggregate2 } })
{
Key = "40127",
DocCount = 1
}
}
}}};
var singleBucketAggregate = new SingleBucketAggregate(backingDictionary);
IReadOnlyDictionary<string, IAggregate> backingDictionary = new Dictionary<string, IAggregate>
{
{ "count_stats", singleBucketAggregate }
};
return new AggregateDictionary(backingDictionary);

scroll_id returns 0 hits when used the second time

I have this code to get the scroll_id after doing the first search:
var initSearch = client.LowLevel.Search<dynamic>(INDEX, TYPE, QUERY, x => x.AddQueryString("scroll", "1m").AddQueryString("size", "2"));
string scrollId = initSearch.Body["_scroll_id"].ToString();
then I used the scrollid during the 2nd search but it didn't return any hits
var scrollSearch = client.LowLevel.ScrollGet<dynamic>(x =>
x.AddQueryString("scroll", "1m").AddQueryString("scroll_id", scrollId));
scrollId = scrollSearch.Body["_scroll_id"].ToString();
var searchHits = int.Parse(scrollSearch.Body["hits"]["total"].ToString());
searchHits.Count is zero. What may be the cause of this? Also, when I loop into the scrollSearch again, I am expecting that the scrollid would change but it is not changing values.
A size of 2 will return 2 documents in each response, including the first response. So, if the total documents for a given query were less than or equal to 2, all documents would be returned within the first response. Take the following for example
void Main()
{
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var defaultIndex = "messages";
var connectionSettings = new ConnectionSettings(pool)
.DefaultIndex(defaultIndex)
.PrettyJson()
.DisableDirectStreaming()
.OnRequestCompleted(response =>
{
if (response.RequestBodyInBytes != null)
{
Console.WriteLine(
$"{response.HttpMethod} {response.Uri} \n" +
$"{Encoding.UTF8.GetString(response.RequestBodyInBytes)}");
}
else
{
Console.WriteLine($"{response.HttpMethod} {response.Uri}");
}
Console.WriteLine();
if (response.ResponseBodyInBytes != null)
{
Console.WriteLine($"Status: {response.HttpStatusCode}\n" +
$"{Encoding.UTF8.GetString(response.ResponseBodyInBytes)}\n" +
$"{new string('-', 30)}\n");
}
else
{
Console.WriteLine($"Status: {response.HttpStatusCode}\n" +
$"{new string('-', 30)}\n");
}
});
var client = new ElasticClient(connectionSettings);
if (client.IndexExists(defaultIndex).Exists)
{
client.DeleteIndex(defaultIndex);
}
client.IndexMany(new[]
{
new Message { Content = "message 1" },
new Message { Content = "message 2" },
new Message { Content = "message 3" },
new Message { Content = "message 4" },
new Message { Content = "message 5" },
new Message { Content = "message 6" },
});
client.Refresh(defaultIndex);
var searchResponse = client.Search<Message>(s => s
.Scroll("1m")
.Size(2)
.Query(q => q
.Terms(t => t
.Field(f => f.Content.Suffix("keyword"))
.Terms("message 1", "message 2")
)
)
);
searchResponse = client.Scroll<Message>("1m", searchResponse.ScrollId);
}
public class Message
{
public string Content { get; set; }
}
The search and scroll responses return
------------------------------
POST http://localhost:9200/messages/message/_search?pretty=true&scroll=1m
{
"size": 2,
"query": {
"terms": {
"content.keyword": [
"message 1",
"message 2"
]
}
}
}
Status: 200
{
"_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAADGFnM1SnhtUVdIUmgtM1YyZ2NQei1hZEEAAAAAAAAAxxZzNUp4bVFXSFJoLTNWMmdjUHotYWRBAAAAAAAAAMgWczVKeG1RV0hSaC0zVjJnY1B6LWFkQQAAAAAAAADJFnM1SnhtUVdIUmgtM1YyZ2NQei1hZEEAAAAAAAAAyhZzNUp4bVFXSFJoLTNWMmdjUHotYWRB",
"took" : 4,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 0.6931472,
"hits" : [
{
"_index" : "messages",
"_type" : "message",
"_id" : "AV8IkTSbM7nzQBTCbQok",
"_score" : 0.6931472,
"_source" : {
"content" : "message 1"
}
},
{
"_index" : "messages",
"_type" : "message",
"_id" : "AV8IkTSbM7nzQBTCbQol",
"_score" : 0.6931472,
"_source" : {
"content" : "message 2"
}
}
]
}
}
------------------------------
POST http://localhost:9200/_search/scroll?pretty=true
{
"scroll": "1m",
"scroll_id": "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAADGFnM1SnhtUVdIUmgtM1YyZ2NQei1hZEEAAAAAAAAAxxZzNUp4bVFXSFJoLTNWMmdjUHotYWRBAAAAAAAAAMgWczVKeG1RV0hSaC0zVjJnY1B6LWFkQQAAAAAAAADJFnM1SnhtUVdIUmgtM1YyZ2NQei1hZEEAAAAAAAAAyhZzNUp4bVFXSFJoLTNWMmdjUHotYWRB"
}
Status: 200
{
"_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAADGFnM1SnhtUVdIUmgtM1YyZ2NQei1hZEEAAAAAAAAAxxZzNUp4bVFXSFJoLTNWMmdjUHotYWRBAAAAAAAAAMgWczVKeG1RV0hSaC0zVjJnY1B6LWFkQQAAAAAAAADJFnM1SnhtUVdIUmgtM1YyZ2NQei1hZEEAAAAAAAAAyhZzNUp4bVFXSFJoLTNWMmdjUHotYWRB",
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 2,
"max_score" : 0.6931472,
"hits" : [ ]
}
}
------------------------------
Since there are only 2 matching documents for the given query and size was set to 2, both documents are returned in the first response and the following scroll response does not contain any hits.
You can use the total from the initial search response to determine whether you need to call the scroll API for more documents.
The actual _scroll_id value is an implementation detail which may or may not change values on subsequent calls. I would not recommend basing any logic on its value, but only use the _scroll_id value returned from the last scroll response in the subsequent scroll request.

Create Spring Data Aggregation Query with Projection of Nested Array

Here is how my document looks like:
{
"_id" : ObjectId("583cb6bcce047d1e68339b64"),
"variantDetails" : [
{
"variants" : {
"_" : "_"
},
"sku" : "069563-59690"
},
{
"variants" : {
"size" : "35"
},
"sku" : "069563-59690-35",
"barcode" : "809702246941"
},
{
"variants" : {
"size" : "36"
},
"sku" : "069563-59690-36",
"barcode" : "809702246958"
}
......
] }
And I would like to use a complex aggregation query like this:
db.getCollection('product').aggregate([
{ '$match': { 'variantDetails.sku': { '$in': ['069563-59690', '069563-59690-36', '069563-59690-37', '511534-01001'] } } },
{ '$project': {'_id': 1, 'variantDetails': 1, 'variantLength': { '$size': '$variantDetails' } } },
{ '$unwind': '$variantDetails' },
{ '$match': { 'variantDetails.sku': { '$in': ['069563-59690', '069563-59690-36', '069563-59690-37', '511534-01001'] } } },
{ '$match': { '$or': [
{'variantLength': { '$ne': 1 }, 'variantDetails.variants._': { '$ne': '_' } },
{'variantLength': 1 }
] } },
{ '$group': { '_id': '$_id', 'variantDetails': { '$push': '$variantDetails' } } },
{ '$project': {'_id': 1, 'variantDetails.sku': 1, 'variantDetails.barcode': 1} }
])
And here is my java code:
final Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("variantDetails.sku").in(skus)),
Aggregation.project("_id", "variantDetails").and("variantDetails").project("size").as("variantLength"),
Aggregation.unwind("variantDetails"),
Aggregation.match(Criteria.where("variantDetails.sku").in(skus)),
Aggregation.match(new Criteria().orOperator(Criteria.where("variantLength").is(1), Criteria.where("variantLength").ne(1).and("variantDetails.variants._").is("_"))),
Aggregation.group("_id").push("variantDetails").as("variantDetails"),
Aggregation.project("_id", "variantDetails.sku", "variantDetails.barcode")
);
final AggregationResults<Product> result = this.mongo.aggregate(agg, this.mongo.getCollectionName(Product.class), Product.class);
return result.getMappedResults();
The problem is that spring translate
Aggregation.project("_id", "variantDetails.sku", "variantDetails.barcode")
To
{ "$project" : { "_id" : 1 , "sku" : "$variantDetails.sku" , "barcode" : "$variantDetails.barcode"}
But I'm expecting
{ '$project': {'_id': 1, 'variantDetails.sku': 1, 'variantDetails.barcode': 1} }
Could someone let me know how to make it right?
I had the same issue and this way works:
Aggregation.project("_id")
.andExpression("variantDetails.sku").as("variantDetails.sku")
.andExpression("variantDetails.barcode").as("variantDetails.barcode"));
The projection will be:
{'$project': {'_id': 1, 'variantDetails.sku': '$variantDetails.sku',
'variantDetails.barcode': '$variantDetails.barcode'} }
You just need to specify the label as alias in the projection operation as the default that spring provides doesnt match. Use Spring 1.8.5 version
Aggregation.project("_id")
.and(context -> new BasicDBObject("$arrayElemAt", Arrays.asList("variantDetails.sku", 0))).as("variantDetails.sku")
.and(context -> new BasicDBObject("$arrayElemAt", Arrays.asList("variantDetails.barcode", 0))).as("variantDetails.barcode"));
May be an old question, but I faced the same issue pointed by Sean.
If found that if you want the expected result
{ '$project': {'_id': 1, 'variantDetails.sku': 1, 'variantDetails.barcode': 1} }
a solution can be:
Aggregation.project("_id")
.andExpression("1").as("variantDetails.sku")
.andExpression("1").as("variantDetails.barcode")
Virginia León's answer was the starting point for finding this solution

Compare data element with partial mean in d3

I have the following data
[{"devcount" : 1 , "dayofweek" :0, "hour" : 1 },
{"devcount" : 2 , "dayofweek" :0, "hour" : 2 },
{"devcount" : 3 , "dayofweek" :1, "hour" : 2 },
{"devcount" : 4 , "dayofweek" :1, "hour" : 3 },
{"devcount" : 6 , "dayofweek" :1, "hour" : 4 },
{"devcount" : 5 , "dayofweek" :1, "hour" : 5 },
{"devcount" : 7 , "dayofweek" :2, "hour" : 5 },
{"devcount" : 8 , "dayofweek" :2, "hour" : 6 },
{"devcount" : 9 , "dayofweek" :2, "hour" : 7 },
{"devcount" : 10 , "dayofweek" :2, "hour" : 9 }]
It is required to compare the devcount with the group average of devcount for each dayofweek.
i.e. for the fist row, devcount=1 is to be compared with the the average device count for the dayofweek-0 (= 1.5) and "yes" to be returned if the devcount is lesser. Else "No" should be returned.
I have coded as below.
smry=d3.nest()
.key( function(d) { return d.dayofweek;})
.rollup(function(d) {return d3.mean(d, function(g) {return g.devcount; })})
.entries(result);
I am not sure how to compare the smry data and the original data.
The original data will be used in selectAll for creating rectangles and the output after comparison needs for determining the colour of the rectangle
You can do it as shown in the snippet below.
test = [{
"devcount": 1,
"dayofweek": 0,
"hour": 1
}, {
"devcount": 2,
"dayofweek": 0,
"hour": 2
},
{
"devcount": 3,
"dayofweek": 1,
"hour": 2
}, {
"devcount": 4,
"dayofweek": 1,
"hour": 3
}, {
"devcount": 6,
"dayofweek": 1,
"hour": 4
}, {
"devcount": 5,
"dayofweek": 1,
"hour": 5
},
{
"devcount": 7,
"dayofweek": 2,
"hour": 5
}, {
"devcount": 8,
"dayofweek": 2,
"hour": 6
}, {
"devcount": 9,
"dayofweek": 2,
"hour": 7
}, {
"devcount": 10,
"dayofweek": 2,
"hour": 9
}
];
//make the summary using nest
smry = d3.nest()
.key(function(d) {
return d.dayofweek;
})
.rollup(function(d) {
return d3.mean(d, function(g) {
return g.devcount;
})
})
.entries(test);
test.forEach(function(t) {
//find the value from summary for dayofweek
var k = smry.find(function(s) {
return (s.key == t.dayofweek)
});
//check the day of week with the mean, set the flag in the data
if(k.values<t.devcount){
t.flag = true;
} else {
t.flag = false;
}
});
console.log(test);//this now has the flag to determine the color
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.4.11/d3.min.js"></script>

NVD3 Distinct Bar with "Duplicate Labels"

The xaxis on our Distinct Bar Chart is user names. The problem I have is that there can be multiple users with the same name.
I want the x-axis to be distinct by the user's id but would like the label to be the user's name (Yeah, I understand that the name is ambiguous, but that is OK for our purposes)
Given the following data how would I get the x-axis to be distinct by id but display the label value
[{
"key" : "tester",
"values" : [{
"value" : 5.0,
"label" : "John Smith",
"color" : "#9BA474",
"id" : 1388
}, {
"value" : 10.25,
"label" : "Jane D'oh",
"color" : "#356AA0",
"id" : 11
}, {
"value" : 3.5,
"label" : "John Smith",
"color" : "red",
"id" : 12
}]
}]
I may have the answer to my own question. If there are other ways of doing this I'd like to hear them.
var chart = nv.models.discreteBarChart()
.x(function(d, i) { return i })
.y(function(d) { return d.value });
chart.xAxis
.tickFormat(function(d) {return data[0].values[d].label; });

Resources