Spring mongoDB aggregation group on date minute subset - spring

I'm running Spring 2.2.7 with spring-data-mongodb.
I've an entity called BaseSample stored in a samples MongoDb collection and I want to group records by minute from the created date and getting the average value collected. I don't know how to use the DateOperators.Minute in the group aggregation operation.
detailed explanation
#Data
#Document(collection = "samples")
#EqualsAndHashCode
public class BaseSample extends Message {
private float value;
private UdcUnitEnum unit;
private float accuracy;
public LocalDateTime recorded;
}
that extends a Message class
#Data
#EqualsAndHashCode
public class Message {
#Indexed
private String sensorUuId;
#Indexed
private String fieldUuId;
#Indexed
private String baseStationUuId;
#Indexed
private Date created;
}
on which I want to apply this query
db.samples.aggregate ([
{
$match: {
$and : [ {fieldUuId:"BS1F1"}, {unit: "DGC"}, {created : {$gte : ISODate("2020-05-30T17:00:00.0Z")}}, {created : {$lt : ISODate("2020-05-30T17:15:00.0Z")}}]
}
},
{
$group: {
_id : { $minute : "$created"},
date : {$first : {$dateToString:{date: "$created", format:"%Y-%m-%d"}}},
time : {$first : {$dateToString:{date: "$created", format:"%H:%M"}}},
unit : {$first : "$unit"},
data : { $avg : "$value"}
}
},
{
$sort: {date:1, time:1}
}
])
on the samples collection (exerpt)
{
"_id" : ObjectId("5ed296150af58a1c60c4f154"),
"value" : 90.85242462158203,
"unit" : "HPA",
"accuracy" : 0.6498473286628723,
"recorded" : ISODate("2020-05-30T17:21:25.850Z"),
"sensorUuId" : "458f0ffd-13f9-466d-81a1-8d2e1c808da9",
"fieldUuId" : "BS1F2",
"baseStationUuId" : "BS1",
"created" : ISODate("2020-05-30T17:21:25.777Z"),
"_class" : "org.open_si.udc_common.models.BaseSample"
}
{
"_id" : ObjectId("5ed296150af58a1c60c4f155"),
"value" : 40.84038162231445,
"unit" : "HPA",
"accuracy" : 0.030185099691152573,
"recorded" : ISODate("2020-05-30T17:21:25.950Z"),
"sensorUuId" : "b396264f-fcd5-4653-8ac8-358ca3a4cb87",
"fieldUuId" : "BS2F3",
"baseStationUuId" : "BS2",
"created" : ISODate("2020-05-30T17:21:25.868Z"),
"_class" : "org.open_si.udc_common.models.BaseSample"
}
I coded the following method to get the average value of samples grouped per minute for a selected unit type (degree, ...) in a selected field (sensors logical group)
public List aggregateFromField(String fieldUuId, UdcUnitEnum unit, LocalDateTime from, LocalDateTime to, Optional pageNumber, Optional pageSize){
Pageable paging = new PagingHelper(pageNumber, pageSize).getPaging();
MatchOperation fieldMatch = Aggregation.match(Criteria.where("fieldUuId").is(fieldUuId));
MatchOperation unitMatch = Aggregation.match(Criteria.where("unit").is(unit.name()));
MatchOperation fromDateMatch = Aggregation.match(Criteria.where("created").gte(from));
MatchOperation toDateMatch = Aggregation.match(Criteria.where("created").lt(to));
DateOperators.Minute minute = DateOperators.Minute.minuteOf("created");
GroupOperation group = Aggregation.group("created")
.first("created").as("date")
.first("created").as("time")
.first("unit").as("unit")
.avg("value").as("avg")
;
SortOperation sort = Aggregation.sort(Sort.by(Sort.Direction.ASC, "$date")).and(Sort.by(Sort.Direction.ASC, "$time"));
SkipOperation skip = Aggregation.skip(paging.getOffset());
LimitOperation limit = Aggregation.limit(paging.getPageSize());
Aggregation agg = Aggregation.newAggregation(
fieldMatch,
unitMatch,
fromDateMatch,
toDateMatch,
group,
sort,
skip,
limit
);
AggregationResults<SampleAggregationResult> results = mongoTemplate.aggregate(agg, mongoTemplate.getCollectionName(BaseSample.class), SampleAggregationResult.class);
return results.getMappedResults();
}
this result class is :
#Data
public class SampleAggregationResult {
private String date;
private String time;
private String unit;
private float data;
}
Any idea on using the DateOperators.Minute type in the agrregation group operation ?
thnaks in advance.

Related

Update a list of objects in spring mongodb

In the below code I want to generate addrId automatically and show it in the Person document, but the addrId is not showing up in the document.
#Document
public class Person {
#Id
String id;
List<Address> addresses;
}
public class Address {
#Id
String addrId;
String street;
}
public class Example {
public Person createAddress(Person person, Address addr) {
Set<Address> addresses = new HashSet<>();
addresses.add(addr);
Query query = new Query();
query.addCriteria(Criteria.where("id").is(id));
Person person = mongoTemplate.findOne(query, Person.class);
person.setAddresses(addresses);
return mongoTemplate.save(person);
}
}
Expected document with addrId:
{
"_id" : ObjectId("592c7029aafef820f432c5f3"),
"_class" : "tutorial.mongodb.documents.Person",
"addresses" : [{
"addrId" : ObjectId("321c7029aafed220f432d321"),
"street" : "London street"
}]
}
but addrId is not getting displayed as seen in the below document:
{
"_id" : ObjectId("592c7029aafef820f432c5f3"),
"_class" : "tutorial.mongodb.documents.Person",
"addresses" : [{
"street" : "London street"
}]
}
This works for me as expected:
public String createAddress(Person person, Address addr) {
addr.setAddrId(String.valueOf(UUID.randomUUID()));
List<Address > addrs = person.getAddresses();
addrs.add(addr);
person.setAddresses(addrs);
mongoTemplate.save(person);
return addr.getAddrId();
}

ElasticsearchRepository skip null values

I have the Repo to interact with ES index:
#Repository
public interface RegDocumentRepo extends ElasticsearchRepository<RegDocument, String> {
}
RegDocument class is a POJO of reg-document index:
#Document(indexName = "reg-document")
#Data
#AllArgsConstructor
#NoArgsConstructor
public class RegDocument {
#Id
String id;
#Field(type = FieldType.Nested, includeInParent = true)
private List<Map<String, Object>> attachments;
private String author;
#Field(type = FieldType.Nested, includeInParent = true)
private List<Map<String, Object>> classification;
private String content;
private String intent;
#Field(type = FieldType.Nested, includeInParent = true)
private List<Map<String, Object>> links;
private String name;
#Field(name = "publication_date")
private String publicationDate;
private Integer raiting;
private Long status;
private String title;
private String type;
private String version;
}
To hide my business-logic I use service:
#RequiredArgsConstructor
#Service
public class SearchServiceImpl {
#Autowired
RegDocumentRepo regDocumentRepo;
public RegDocument updateRating(String uuid, Integer rating) throws IOException {
final RegDocument regDocument = regDocumentRepo
.findById(uuid)
.orElseThrow(() -> new IOException(String.format("No document with %s id", uuid)));
Integer ratingFromDB = regDocument.getRaiting();
ratingFromDB = ratingFromDB == null ? rating : ratingFromDB + rating;
regDocument.setRaiting(ratingFromDB);
final RegDocument save = regDocumentRepo.save(regDocument);
return save;
}
}
So I had the such document in my ES index:
{
"_index" : "reg-document",
"_type" : "_doc",
"_id" : "9wEgQnQBKzq7IqBZMDaO",
"_score" : 1.0,
"_source" : {
"raiting" : null,
"attachments" : null,
"author" : null,
"type" : "answer",
"classification" : [
{
"code" : null,
"level" : null,
"name" : null,
"description" : null,
"id_parent" : null,
"topic_type" : null,
"uuid" : null
}
],
"intent" : null,
"version" : null,
"content" : "В 2019 году размер материнского капитала составляет 453026 рублей",
"name" : "Каков размер МСК в 2019 году?",
"publication_date" : "2020-08-26 06:49:10",
"rowkey" : null,
"links" : null,
"status" : 1
}
}
But after I update my ranking score, I have next structure:
{
"_index" : "reg-document",
"_type" : "_doc",
"_id" : "9wEgQnQBKzq7IqBZMDaO",
"_score" : 1.0,
"_source" : {
"raiting" : 4,
"type" : "answer",
"classification" : [
{
"code" : null,
"level" : null,
"name" : null,
"description" : null,
"id_parent" : null,
"topic_type" : null,
"uuid" : null
}
],
"content" : "В 2019 году размер материнского капитала составляет 453026 рублей",
"name" : "Каков размер МСК в 2019 году?",
"publication_date" : "2020-08-26 06:49:10",
"status" : 1
}
}
As you can see, Java service skip NULL values. But if the field is nested, null values were saved.
ElasticSearch version - 7.8.0
maven dependency for spring-data:
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<version>4.0.0.RELEASE</version>
</dependency>
So how can i SAVE null values, not skip them?
**
UDP
**
I have investigated spring-data-elasticsearch-4.0.0 dependency and find out, as Best Answer author said, that MappingElasticsearchConverter.java has following methods:
#Override
public void write(Object source, Document sink) {
Assert.notNull(source, "source to map must not be null");
if (source instanceof Map) {
// noinspection unchecked
sink.putAll((Map<String, Object>) source);
return;
}
Class<?> entityType = ClassUtils.getUserClass(source.getClass());
TypeInformation<?> type = ClassTypeInformation.from(entityType);
if (requiresTypeHint(type, source.getClass(), null)) {
typeMapper.writeType(source.getClass(), sink);
}
Optional<Class<?>> customTarget = conversions.getCustomWriteTarget(entityType, Map.class);
if (customTarget.isPresent()) {
sink.putAll(conversionService.convert(source, Map.class));
return;
}
ElasticsearchPersistentEntity<?> entity = type.getType().equals(entityType)
? mappingContext.getRequiredPersistentEntity(type)
: mappingContext.getRequiredPersistentEntity(entityType);
writeEntity(entity, source, sink, null);
}
This methods explain why nested data was saved as null and wasn't skip. It just put Map inside.
So the next method use reflection in such way. So if it is a null value, it's just skip it:
protected void writeProperties(ElasticsearchPersistentEntity<?> entity, PersistentPropertyAccessor<?> accessor,
MapValueAccessor sink) {
for (ElasticsearchPersistentProperty property : entity) {
if (!property.isWritable()) {
continue;
}
Object value = accessor.getProperty(property);
if (value == null) {
continue;
}
if (property.hasPropertyConverter()) {
ElasticsearchPersistentPropertyConverter propertyConverter = property.getPropertyConverter();
value = propertyConverter.write(value);
}
if (!isSimpleType(value)) {
writeProperty(property, value, sink);
} else {
Object writeSimpleValue = getWriteSimpleValue(value);
if (writeSimpleValue != null) {
sink.set(property, writeSimpleValue);
}
}
}
}
There is no official solution. So i have created a Jira ticket
The null values of the inner objects are stored, because this happens when the Map with null values for keys is stored.
Entity properties with a null value are not persisted by Spring Data Elasticsearch are not persisted as this it would store information that is not needed for saving/retrieving the data.
If you need the null values to be written, this would mean, that we'd need to add some flag to the #Field annotation for this, can you add an issue in Jira (https://jira.spring.io/projects/DATAES/issues) for this?
Edit: Implemented in versions 4.0.4.RELEASE and 4.1.0.RC1

Not able to get composition pojo values in spring mongo

This is my pojo structure which is having a Map as an instance variable
#Document(collection = "notifications")
public class PushNotification {
private String notificationId;
private String title;
private String message;
private PushType pushType;
private PushState pushState;
private String stateData;
private String customIconUrl;
private String bannerImageUrl;
private Set<Long> customerIdSet;
private int expiryTime = 60 * 60 * 24;
private Date pushDateCreated;
private Map<String, PushNotificationBatch> pushBatch;
This is how my pojo stored in Mongo (With PushNotificationBatch value)
{
"_id" : "56",
"_class" : "com.medplus.customer.domain.PushNotification",
"title" : "Mongo DashBoard",
"message" : "Mongo DashBoard",
"pushType" : "REGISTERED_CUSTOMER",
"pushState" : "QUICK_ORDER",
"bannerImageUrl" : "http://127.0.0.1/masters/pushNotification/201611/NOTIF_BANNER_20161130174814417",
"customerIdSet" : [
NumberLong(28716065),
NumberLong(22203194),
NumberLong(28716069),
NumberLong(28715739),
NumberLong(25403900),
NumberLong(28715715),
NumberLong(28716073)
],
"expiryTime" : 172800,
"pushDateCreated" : ISODate("2016-11-30T12:18:18.448Z"),
"pushBatch" : {
"0" : {
"uuid" : "96b2a850-1f92-4f75-924f-2787127ec226",
"batchStatus" : "SUCCESS"
}
}
}
This is the response i'm getting which doesn't contain my Map variable
[PushNotification [notificationId=54
title=Mongo 4
message=Mongo 4
pushType=REGISTERED_CUSTOMER
pushState=REWARD
stateData=null
customIconUrl=null
bannerImageUrl=http://127.0.0.1/masters/imgUrl
customerIdSet=[27796308
28716083]
expiryTime=7200
pushDateCreated=Wed Nov 30 11:28:56 IST 2016]"

Programmatic mapping with embedded index in Hibernate Search results in unable to find field error

I am having to do a programmatic configuration of the fields to be indexed with Hibernate Search.
In the scenario below the use of indexEmbedded() is resulting in a "field not found error".
#Entity
public class AT {
#ManyToOne(fetch = FetchType.EAGER)
#JoinColumn(name = "ARRM_IDE", nullable = false)
private A arr;
private Date dateType;
(and other fields)
}
#Entity
public class A {
#Id
#SequenceGenerator(name = "C_SEQUENCE", sequenceName = "S_ARRM_01")
#GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "C_SEQUENCE")
#Column(name = "IDE_ARR")
private Long id;
}
SearchMapping mapping = new SearchMapping();
mapping.entity(AT.class).indexed()
.property("dateType", ElementType.FIELD)
.field()
.store(Store.YES)
.property("arr", ElementType.FIELD)
.indexEmbedded()
.entity(A.class).indexed()
.property("id", ElementType.FIELD).documentId().name("arrId")
.field()
.store(Store.YES)
;
When I create and persist entities (I have integrated Hibernate Search with Elasticsearch), the entities are created and the indexes are created in Elasticsearch also.
contents on Elasticsearch:
"_index" : "com.....at",
"_type" : "com.....AT",
"_id" : "7744",
"_score" : 1.0,
"_source" : {
"dateType" : "2016-06-12T06:08:52.780Z",
"arr" : {
"id" : 6352
}
}
} ]
But when I try querying using the Hibernate Search Lucene query it fails:
FullTextEntityManager fullTextEntityManager =
org.hibernate.search.jpa.Search.getFullTextEntityManager(em);
QueryBuilder qb = fullTextEntityManager.getSearchFactory()
.buildQueryBuilder().forEntity(AT.class).get();
org.apache.lucene.search.Query luceneQuery = qb.bool()
.must(qb
.range()
.onField("dateType")
.from(parseDate(startDate))
.to(parseDate(endDate)).excludeLimit()
.createQuery())
.must(qb
.keyword()
.onField("arr")
.matching(crsArrId).createQuery())
.createQuery();
Sort sort = null;
if (order == OrderEnum.ASCENDING) {
sort = new Sort(
new SortField("dateType", SortField.Type.STRING));
} else {
sort = new Sort(
new SortField("dateType", SortField.Type.STRING, true));
}
FullTextQuery jpaQuery = fullTextEntityManager.createFullTextQuery(luceneQuery, AT.class);
jpaQuery.setSort(sort);
jpaQuery.setFirstResult(offset);
jpaQuery.setMaxResults(maxReturnedEvents);
return jpaQuery.getResultList();
Error is:
org.hibernate.search.exception.SearchException: Unable to find field arr in com....AT
at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.objectToString(DocumentBuilderIndexedEntity.java:977)
at org.hibernate.search.query.dsl.impl.FieldContext.objectToString(FieldContext.java:75)
at org.hibernate.search.query.dsl.impl.ConnectedMultiFieldsTermQueryBuilder.buildSearchTerm(ConnectedMultiFieldsTermQueryBuilder.java:145)
at org.hibernate.search.query.dsl.impl.ConnectedMultiFieldsTermQueryBuilder.createQuery(ConnectedMultiFieldsTermQueryBuilder.java:105)
at org.hibernate.search.query.dsl.impl.ConnectedMultiFieldsTermQueryBuilder.createQuery(ConnectedMultiFieldsTermQueryBuilder.java:67)
Thanks for your help!
You want to search on arr.id, not arr.
Just change .onField("arr") to .onField("arr.id").

Spring: one JPA model, many JSON respresentations

I'm writing a RESTful web service using Spring/JPA. There's a JPA model which is exposed through the web service. The 'Course' model is quite spacious - it actually is composed of several sets of data: general information, pricing details and some caches.
The issue I encounter is the inability to issue different JSON representations using the same JPA model.
The in first case I only need to return general_info set of data for courses:
GET /api/courses/general_info
in the second case I would like to return pricing set of data only:
GET /api/courses/pricing
I see the following ways to solve this, not in particular order:
To create CourseGeneralInfo and CoursePricing JPA models using
the origin database table as a source. CourseGeneralInfo model
would have its own set of fields and CoursePricing would have its
own ones. This way I would have the JSON I need.
To refactor the stuff out of the Course model/table to have
GeneralInfo and PricingDetails to be separate JPA entities. Ok, this sounds like the best one (imo) though the database is legacy and it is not something I can change easily...
Leverage some sort of DTO and Spring Mappers to convert the JPA model to representation needed in any particular case.
What approach would you recommend?
I was just reading about some really nifty features in Spring 4.1, which allow you to use different views via annotations.
from: https://spring.io/blog/2014/12/02/latest-jackson-integration-improvements-in-spring
public class View {
interface Summary {}
}
public class User {
#JsonView(View.Summary.class)
private Long id;
#JsonView(View.Summary.class)
private String firstname;
#JsonView(View.Summary.class)
private String lastname;
private String email;
private String address;
private String postalCode;
private String city;
private String country;
}
public class Message {
#JsonView(View.Summary.class)
private Long id;
#JsonView(View.Summary.class)
private LocalDate created;
#JsonView(View.Summary.class)
private String title;
#JsonView(View.Summary.class)
private User author;
private List<User> recipients;
private String body;
}
Thanks to Spring MVC #JsonView support, it is possible to choose, on a per handler method basis, which field should be serialized:
#RestController
public class MessageController {
#Autowired
private MessageService messageService;
#JsonView(View.Summary.class)
#RequestMapping("/")
public List<Message> getAllMessages() {
return messageService.getAll();
}
#RequestMapping("/{id}")
public Message getMessage(#PathVariable Long id) {
return messageService.get(id);
}
}
In this example, if all messages are retrieved, only the most important fields are serialized thanks to the getAllMessages() method annotated with #JsonView(View.Summary.class):
[ {
"id" : 1,
"created" : "2014-11-14",
"title" : "Info",
"author" : {
"id" : 1,
"firstname" : "Brian",
"lastname" : "Clozel"
}
}, {
"id" : 2,
"created" : "2014-11-14",
"title" : "Warning",
"author" : {
"id" : 2,
"firstname" : "Stéphane",
"lastname" : "Nicoll"
}
}, {
"id" : 3,
"created" : "2014-11-14",
"title" : "Alert",
"author" : {
"id" : 3,
"firstname" : "Rossen",
"lastname" : "Stoyanchev"
}
} ]
In Spring MVC default configuration, MapperFeature.DEFAULT_VIEW_INCLUSION is set to false. That means that when enabling a JSON View, non annotated fields or properties like body or recipients are not serialized.
When a specific Message is retrieved using the getMessage() handler method (no JSON View specified), all fields are serialized as expected:
{
"id" : 1,
"created" : "2014-11-14",
"title" : "Info",
"body" : "This is an information message",
"author" : {
"id" : 1,
"firstname" : "Brian",
"lastname" : "Clozel",
"email" : "bclozel#pivotal.io",
"address" : "1 Jaures street",
"postalCode" : "69003",
"city" : "Lyon",
"country" : "France"
},
"recipients" : [ {
"id" : 2,
"firstname" : "Stéphane",
"lastname" : "Nicoll",
"email" : "snicoll#pivotal.io",
"address" : "42 Obama street",
"postalCode" : "1000",
"city" : "Brussel",
"country" : "Belgium"
}, {
"id" : 3,
"firstname" : "Rossen",
"lastname" : "Stoyanchev",
"email" : "rstoyanchev#pivotal.io",
"address" : "3 Warren street",
"postalCode" : "10011",
"city" : "New York",
"country" : "USA"
} ]
}
Only one class or interface can be specified with the #JsonView annotation, but you can use inheritance to represent JSON View hierarchies (if a field is part of a JSON View, it will be also part of parent view). For example, this handler method will serialize fields annotated with #JsonView(View.Summary.class) and #JsonView(View.SummaryWithRecipients.class):
public class View {
interface Summary {}
interface SummaryWithRecipients extends Summary {}
}
public class Message {
#JsonView(View.Summary.class)
private Long id;
#JsonView(View.Summary.class)
private LocalDate created;
#JsonView(View.Summary.class)
private String title;
#JsonView(View.Summary.class)
private User author;
#JsonView(View.SummaryWithRecipients.class)
private List<User> recipients;
private String body;
}
#RestController
public class MessageController {
#Autowired
private MessageService messageService;
#JsonView(View.SummaryWithRecipients.class)
#RequestMapping("/with-recipients")
public List<Message> getAllMessagesWithRecipients() {
return messageService.getAll();
}
}
In Spring Data REST 2.1 there is a new mechanism for this purpose - Projections (It's now part of spring-data-commons).
You'll need to define interface, containing exactly exposed fields:
#Projection(name = "summary", types = Course.class)
interface CourseGeneralInfo {
GeneralInfo getInfo();
}
After that Spring will be able to find it automagically in your source, and you could make requests to your existing endpoints, like this:
GET /api/courses?projection=general_info
Based on
https://spring.io/blog/2014/05/21/what-s-new-in-spring-data-dijkstra
Spring sample project with projections:
https://github.com/spring-projects/spring-data-examples/tree/master/rest/projections

Resources