How can I read Flux<DataBuffer> content? - spring-boot

I want to read mulitpart/formdata, one part is application/JSON. I can't get them to Map<String,String>, Is there any way to parse Part to String?
private Map<String, String> getFormData(String path, MultiValueMap<String, Part> partMultiValueMap) {
if (partMultiValueMap != null) {
Map<String, String> formData = new HashMap<>();
Map<String, Part> multiPartMap = partMultiValueMap.toSingleValueMap();
for (Map.Entry<String, Part> partEntry : multiPartMap.entrySet()) {
Part part = partEntry.getValue();
if (part instanceof FormFieldPart) {
formData.put(partEntry.getKey(), ((FormFieldPart) part).value());
} else {
String bodyString = bufferToStr(part.content());
formData.put(partEntry.getKey(), bodyString);
}
}
return formData;
}
return null;
}
extra Flux
private String bufferToStr(Flux<DataBuffer> content){
AtomicReference<String> res = new AtomicReference<>();
content.subscribe(buffer -> {
byte[] bytes = new byte[buffer.readableByteCount()];
buffer.read(bytes);
DataBufferUtils.release(buffer);
res.set(new String(bytes, StandardCharsets.UTF_8));
});
return res.get();
}
Subscribe is async; bufferToStr value may be null?

You could do it in non-blocking way with StringDecoder
Basically you could write your code to return Mono<Map<>>
Note: I'm using Pair class here to return key-value and later collect them to Map
Pair I'm using here is from package org.springframework.data.util.Pair
public Mono<Map<String, String>> getFormData(MultiValueMap<String, Part> partMultiValueMap) {
Map<String, Part> multiPartMap = partMultiValueMap.toSingleValueMap();
return Flux.fromIterable(multiPartMap.entrySet())
.flatMap(entry -> {
Part part = entry.getValue();
if (part instanceof FormFieldPart) {
return Mono.just(
Pair.of(entry.getKey(), ((FormFieldPart) part).value()) // return Pair
);
} else {
return decodePartToString(part.content()) // decoding DataBuffers to string
.flatMap(decodedString ->
Mono.just(Pair.of(entry.getKey(), decodedString))); // return Pair
}
})
.collectMap(Pair::getFirst, Pair::getSecond); // map and collect pairs to Map<>
}
private Mono<String> decodePartToString(Flux<DataBuffer> dataBufferFlux) {
StringDecoder stringDecoder = StringDecoder.textPlainOnly();
return stringDecoder.decodeToMono(dataBufferFlux,
ResolvableType.NONE,
MimeTypeUtils.TEXT_PLAIN,
Collections.emptyMap()
);
}

Related

Throw exception to parent method in webclient

Is there is a way to throw the exception to parent method in webclient. I am not able to throw the exception to parent method getStores(). What is the way to throw the exception to parent method in below code.
For example:
#Override
public Mono<Stores> getStores(String id) {
Mono<Stores> stores = null;
try {
WebClientRequest<Stores> webClientRequest = new WebClientRequest<>();
webClientRequest.setContentType("application/json");
webClientRequest.setEndPoint("http://localhost:8090/api/stores/eee");
webClientRequest.setPathParam(id);
stores= webClient.getAsynchronousWebClient(webClientRequest, Stores.class);
} catch(MtampWebClientException e) {
System.out.println("Hello");
}
return stores;
}
public <T> Mono<K> getAsynchronousWebClient(WebClientRequest<T> webClientRequest, Class<K> clazz) {
ResponseSpec retrieve = performWebRequest(webClientRequest);
return retrieve.bodyToMono(clazz).doOnNext(response -> webClientResponse(webClientRequest, response));
}
private <T> ResponseSpec performWebRequest(WebClientRequest<T> webClientRequest) {
WebClient client = webClientBuilder();
LinkedMultiValueMap<String, String> map = setHeaders(webClientRequest);
Consumer<HttpHeaders> consumer = it -> it.addAll(map);
ResponseSpec retrieve;
if (null != webClientRequest.getPathParam()) {
retrieve = client.get().uri(webClientRequest.getEndPoint() + "/" + webClientRequest.getPathParam())
.headers(consumer).retrieve();
} else if (null != webClientRequest.getQueryParam()) {
LinkedMultiValueMap<String, String> queryMap = new LinkedMultiValueMap<>();
webClientRequest.getQueryParam().entrySet().stream().forEach(e -> queryMap.add(e.getKey(), e.getValue()));
retrieve = client.get()
.uri(uriBuilder -> uriBuilder.path(webClientRequest.getEndPoint()).queryParams(queryMap).build())
.headers(consumer).retrieve();
} else {
retrieve = client.get().uri(webClientRequest.getEndPoint()).headers(consumer).retrieve();
}
return retrieve;
}
private WebClient webClientBuilder() {
HttpClient httpClient = HttpClient.create().option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000)
.responseTimeout(Duration.ofMillis(5000))
.doOnConnected(conn -> conn.addHandlerLast(new ReadTimeoutHandler(5000, TimeUnit.MILLISECONDS))
.addHandlerLast(new WriteTimeoutHandler(5000, TimeUnit.MILLISECONDS)));
return WebClient.builder().filter(ExchangeFilterFunction.ofResponseProcessor(this::errorHandler))
.clientConnector(new ReactorClientHttpConnector(httpClient)).build();
}
public Mono<ClientResponse> errorHandler(ClientResponse clientResponse) {
if (clientResponse.statusCode().is5xxServerError()) {
return clientResponse.bodyToMono(String.class).flatMap(errorBody -> Mono
.error(new MtampWebClientException(clientResponse.statusCode().toString(), errorBody)));
} else if (clientResponse.statusCode().is4xxClientError()) {
return clientResponse.bodyToMono(String.class).flatMap(errorBody -> Mono
.error(new MtampWebClientException(clientResponse.statusCode().toString(), errorBody)));
} else {
return Mono.just(clientResponse);
}
}

Mono not executing with schedule

I have a Spring webflux app with the below method.
#Override
public Mono<Integer> updateSetting(int orgId, IntegrationDto dto,
Map<String, Object> jsonMap) {
return retrieveServices(dto.getClientId()).flatMap(services -> {
jsonMap.put("service", services);
return categoryRepository.findCategoryIdCountByName("test", orgId)
.flatMap(categoryIdCount -> {
final ServiceDto serviceInput = new ServiceDto();
if (categoryIdCount == 0) {
return inventoryCategoryRepository.save(InventoryCategory.of("test", orgId))
.flatMap(category -> {
return saveServices(serviceInput, orgId, jsonMap,
category.getCategoryId());
});
} else {
// Some Logc here ...
}
});
}).onErrorResume(e -> {
if (e instanceof WebClientResponseException) {
int statusCode = ((WebClientResponseException) e).getRawStatusCode();
throw new LabServiceException("Unable to connect to the service !", statusCode);
}
throw new ServiceException("Error connecting to the service !");
});
}
private Mono<Services> retrieveServices(final String clientId) {
return webClient.get().uri(props.getBaseUrl() + "/api/v1/services")
.retrieve().bodyToMono(Services.class);
}
private Mono<Integer> saveInventories(ServiceInput serviceInput, int orgId, Map<String, Object> jsonMap,
Long categoryId) {
return refreshInventories(serviceInput, orgId, categoryId).flatMap(reponse -> {
return updateSetting(branchId, jsonMap);
});
}
private Mono<Integer> refreshInventories(ServiceInput serviceInput, int orgId, Long categoryId) {
return inventoryRepository.findAllCodesByTypeBranchId(branchId).collectList().flatMap(codes -> {
return retrieveAvailableServices(Optional.of(serviceInput), categoryId).flatMap(services -> {
List<Inventory> inventories = services.stream()
.filter(inventory -> !codes.contains(inventory.getCode()))
.map(inventoryDto -> toInventory(inventoryDto, branchId)).collect(Collectors.toList());
if (inventories.size() > 0) {
return saveAllInventories(inventories).flatMap(response -> {
return Mono.just(orgId);
});
} else {
return Mono.just(orgId);
}
});
});
}
Here, the updateSettig public method is being invoked from a REST call and all gets executed as expected.
Now, I want to execute the same with a different flow as well like a scheduler.
When I invoke from a scheduler also, It works.
updateSetting(orgId, dto, jsonMap).subscribe();
But, I want to wait until the updateSetting gets executed.
So, tried with the code below.
updateSetting(orgId, dto, jsonMap).flatMap(response -> {
////
});
With the above code, updateSetting method gets invoked, but not getting into the retrieveServices.
return retrieveServices(dto.getClientId()).flatMap(services -> {
You always need to subscribe in the end. So your code should be:
updateSetting(orgId, dto, jsonMap).flatMap(response -> {
////
}).subscribe();

How to write the below code in Java 8 using lambda expressions?

I have the following classes:
package com.test.api;
public class Driver {
private String name;
private int age;
private String gender;
// getters, setters
}
package com.test.api;
import java.util.List;
public class Vehicle {
private String name;
private String make;
private int modelyear;
private List<Driver> drivers;
// getters, setters
}
package com.test.api;
public class Data {
private String driverName;
private String vehicleName;
private String gender;
private boolean isDriverEligible;
private boolean isRecentVehicle;
// getters, setters
}
and I am currently doing this in traditional way without lamdas and streams:
private static List<Data> map(List<Vehicle> vehicleList) {
List<Data> dataList = null;
for (Vehicle vehicle : vehicleList) {
if ("BMW".equalsIgnoreCase(vehicle.getName())) {
dataList = new ArrayList<>();
for (Driver driver : vehicle.getDrivers()) {
if (driver.getAge() > 18 && vehicle.getModelyear() > 2016) {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName(vehicle.getName());
data.setDriverEligible(true);
data.setRecentVehicle(true);
dataList.add(data);
}
if (driver.getAge() > 18 && vehicle.getModelyear() < 2016) {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName(vehicle.getName());
data.setDriverEligible(true);
data.setRecentVehicle(false);
dataList.add(data);
}
}
List<Response> response = service.update(dataList, vehicle);
}
}
return dataList;
}
So I wanted to iterate through List of Drivers for each Vehicle and based on the conditions, I need to map some data. And then once all the data is populated to the "Data" List, I am updating it in to a service. Everything looks good but I wanted to refactor it using java 8.How can I do it?
Prior notice: If you are running this against a database, it's HIGHLY RECOMMENDED to do that with some SQL or stored procedures. Messing with data in the application is not considered a good practice unless it's absolutely necessary.
The idea of using stream is to describe filtering and mapping in distinct blocks.
Let's start simple, from the inner loop.
private static List<Data> map(List<Vehicle> vehicleList) {
List<Data> dataList = new ArrayList<>();
vehicleList.stream().filter(vehicle -> "BMW".equalsIgnoreCase(vehicle.getName())).forEach(vehicle -> {
dataList.addAll(
return vehicle.getDrivers().stream()
.filter(driver -> driver.getAge() > 18)
.map(driver -> {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName(vehicle.getName());
data.setDriverEligible(true);
if (vehicle.getModelyear() > 2016) {
data.setRecentVehicle(true);
return data;
}
if (vehicle.getModelyear() < 2016) {
data.setRecentVehicle(false);
return data;
}
return null;
})
.filter(it -> it != null)
.collect(Collectors.toList())
);
List<Response> response = service.update(dataList, vehicle);
});
return dataList;
}
We have replaced for loops with a stream. Since we are going to add it into a list, we can map the elements into target elements, and then collect them and call addAll(.) to append them to the list.
Then we can make it to the outer loop:
private static List<Data> map(List<Vehicle> vehicleList) {
List<Data> dataList = new ArrayList<>();
vehicleList.stream().filter(vehicle -> "BMW".equalsIgnoreCase(vehicle.getName())).forEach(vehicle -> {
dataList.addAll(
return vehicle.getDrivers().stream()
.filter(driver -> driver.getAge() > 18)
.map(driver -> {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName(vehicle.getName());
data.setDriverEligible(true);
if (vehicle.getModelyear() > 2016) {
data.setRecentVehicle(true);
return data;
}
if (vehicle.getModelyear() < 2016) {
data.setRecentVehicle(false);
return data;
}
return null;
})
.filter(it -> it != null)
.collect(Collectors.toList())
);
List<Response> response = service.update(dataList, vehicle);
});
return dataList;
}
By simply replacing the for loop with a stream and substitute it with forEach is enough. We can also filter out the unwanted elements before forEach with filter.
Here's is how I would do it:
static void inJava8(List<Vehicle> vehicleList) {
Predicate<Vehicle> isBMW = vehicle -> "BMW".equalsIgnoreCase(vehicle.getName());
Predicate<Vehicle> isRecentvehicle = vehicle -> vehicle.getModelyear() > 2016;
Map<Boolean, List<Vehicle>> isRecentToVehicleMap = vehicleList.stream()
.filter(isBMW::test)
.collect(Collectors.partitioningBy(isRecentvehicle));
isRecentToVehicleMap.get(true).stream()
.forEach(vehicle -> processVehicle(vehicle, Test::recentBMWDataCOnstructor));
isRecentToVehicleMap.get(false).stream()
.forEach(vehicle -> processVehicle(vehicle, Test::notRecentBMWDataCOnstructor));
}
public static void processVehicle(Vehicle vehicle, Function<Driver, Data> function) {
// this predicate should be at class level to avoid creating per vehicle
Predicate<Driver> isAgeEligile = driver -> driver.getAge() > 18;
List<Data> dataList = vehicle.getDrivers()
.stream()
.filter(isAgeEligile::test)
.map(function::apply)
.collect(Collectors.toList());
service.update(dataList, vehicle);
}
public static Data recentBMWDataCOnstructor(Driver driver) {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName("BMW");
data.setDriverEligible(true);
data.setRecentVehicle(false);
return data;
}
public static Data notRecentBMWDataCOnstructor(Driver driver) {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName("BMW");
data.setDriverEligible(true);
data.setRecentVehicle(false);
return data;
}
final Vehicle[] vehicle = new Vehicle[1];
vehicleList.stream()
.filter(v -> "BMW".equalsIgnoreCase(v.getName()))
.map(v -> {vehicle[0] = v; return v;})
.flatMap(v -> v.getDrivers().stream())
.filter(driver -> driver.getAge() > 18)
.map(driver -> {
Data data = new Data();
data.setDriverName(driver.getName());
data.setLuxuryVehicle(true);
data.setVehicleName(vehicle[0].getName());
data.setDriverEligible(true);
data.setRecentVehicle(vehicle[0].getModelyear() > 2016);
return data;
})
.collect(Collectors.toList());

AggregatingReplyingKafkaTemplate releaseStrategy Question

There seem to be an issue when I use AggregatingReplyingKafkaTemplate with template.setReturnPartialOnTimeout(true) in that, it returns timeout exception even if partial results are available from consumers.
In example below, I have 3 consumers to reply to the request topic and i've set the reply timeout at 10 seconds. I've explicitly delayed the response of Consumer 3 to 11 seconds, however, I expect the response back from Consumer 1 and 2, so, I can return partial results. However, I am getting KafkaReplyTimeoutException. Appreciate your inputs. Thanks.
I follow the code based on the Unit Test below.
[ReplyingKafkaTemplateTests][1]
I've provided the actual code below:
#RestController
public class SumController {
#Value("${kafka.bootstrap-servers}")
private String bootstrapServers;
public static final String D_REPLY = "dReply";
public static final String D_REQUEST = "dRequest";
#ResponseBody
#PostMapping(value="/sum")
public String sum(#RequestParam("message") String message) throws InterruptedException, ExecutionException {
AggregatingReplyingKafkaTemplate<Integer, String, String> template = aggregatingTemplate(
new TopicPartitionOffset(D_REPLY, 0), 3, new AtomicInteger());
String resultValue ="";
String currentValue ="";
try {
template.setDefaultReplyTimeout(Duration.ofSeconds(10));
template.setReturnPartialOnTimeout(true);
ProducerRecord<Integer, String> record = new ProducerRecord<>(D_REQUEST, null, null, null, message);
RequestReplyFuture<Integer, String, Collection<ConsumerRecord<Integer, String>>> future =
template.sendAndReceive(record);
future.getSendFuture().get(5, TimeUnit.SECONDS); // send ok
System.out.println("Send Completed Successfully");
ConsumerRecord<Integer, Collection<ConsumerRecord<Integer, String>>> consumerRecord = future.get(10, TimeUnit.SECONDS);
System.out.println("Consumer record size "+consumerRecord.value().size());
Iterator<ConsumerRecord<Integer, String>> iterator = consumerRecord.value().iterator();
while (iterator.hasNext()) {
currentValue = iterator.next().value();
System.out.println("response " + currentValue);
System.out.println("Record header " + consumerRecord.headers().toString());
resultValue = resultValue + currentValue + "\r\n";
}
} catch (Exception e) {
System.out.println("Error Message is "+e.getMessage());
}
return resultValue;
}
public AggregatingReplyingKafkaTemplate<Integer, String, String> aggregatingTemplate(
TopicPartitionOffset topic, int releaseSize, AtomicInteger releaseCount) {
//Create Container Properties
ContainerProperties containerProperties = new ContainerProperties(topic);
containerProperties.setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
//Set the consumer Config
//Create Consumer Factory with Consumer Config
DefaultKafkaConsumerFactory<Integer, Collection<ConsumerRecord<Integer, String>>> cf =
new DefaultKafkaConsumerFactory<>(consumerConfigs());
//Create Listener Container with Consumer Factory and Container Property
KafkaMessageListenerContainer<Integer, Collection<ConsumerRecord<Integer, String>>> container =
new KafkaMessageListenerContainer<>(cf, containerProperties);
// container.setBeanName(this.testName);
AggregatingReplyingKafkaTemplate<Integer, String, String> template =
new AggregatingReplyingKafkaTemplate<>(new DefaultKafkaProducerFactory<>(producerConfigs()), container,
(list, timeout) -> {
releaseCount.incrementAndGet();
return list.size() == releaseSize;
});
template.setSharedReplyTopic(true);
template.start();
return template;
}
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,bootstrapServers);
props.put(ConsumerConfig.GROUP_ID_CONFIG, "test_id");
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringDeserializer.class);
return props;
}
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
// list of host:port pairs used for establishing the initial connections to the Kakfa cluster
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
bootstrapServers);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
org.apache.kafka.common.serialization.StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.class);
return props;
}
public ProducerFactory<Integer,String> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
#KafkaListener(id = "def1", topics = { D_REQUEST}, groupId = "D_REQUEST1")
#SendTo // default REPLY_TOPIC header
public String dListener1(String in) throws InterruptedException {
return "First Consumer : "+ in.toUpperCase();
}
#KafkaListener(id = "def2", topics = { D_REQUEST}, groupId = "D_REQUEST2")
#SendTo // default REPLY_TOPIC header
public String dListener2(String in) throws InterruptedException {
return "Second Consumer : "+ in.toLowerCase();
}
#KafkaListener(id = "def3", topics = { D_REQUEST}, groupId = "D_REQUEST3")
#SendTo // default REPLY_TOPIC header
public String dListener3(String in) throws InterruptedException {
Thread.sleep(11000);
return "Third Consumer : "+ in;
}
}
'''
[1]: https://github.com/spring-projects/spring-kafka/blob/master/spring-kafka/src/test/java/org/springframework/kafka/requestreply/ReplyingKafkaTemplateTests.java
template.setReturnPartialOnTimeout(true) simply means the template will consult the release strategy on timeout (with the timeout argument = true, to tell the strategy it's a timeout rather than a delivery call).
It must return true to release the partial result.
This is to allow you to look at (and possibly modify) the list to decide whether you want to release or discard.
Your strategy ignores the timeout parameter:
(list, timeout) -> {
releaseCount.incrementAndGet();
return list.size() == releaseSize;
});
You need return timeout ? true : { ... }.

java 8 nested null check list of objects

I am trying to do a nested null check and then clear the values in map in the nested object if the map is not null.
The following is my hypothetical code. I am wondering if this is the right way to do it or is there a more elegant solution to this.
package exp.myJavaLab.Experiments;
import java.util.*;
public class OptionalTest {
public Inner inner;
public static void main(String[] args) {
OptionalTest testObj = new OptionalTest();
Pojo pojo1 = new Pojo();
pojo1.id = 1;
Map<String, String> dataMap = new HashMap<>();
dataMap.put("a","b");
pojo1.dataMap = dataMap;
Pojo pojo2 = new Pojo();
pojo2.id = 2;
Inner inner = new Inner();
inner.pojoList = Arrays.asList(pojo1, pojo2);
testObj.inner = inner;
System.out.println(testObj);
Optional.ofNullable(testObj.inner)
.map(Inner::getPojoList)
.ifPresent(pojos -> pojos.forEach(pojo -> {
if (pojo.getDataMap() != null) {
pojo.getDataMap().clear();
}
}));
System.out.println(testObj);
}
#Override
public String toString() {
final StringBuilder sb = new StringBuilder("OptionalTest{");
sb.append("inner=").append(inner);
sb.append('}');
return sb.toString();
}
}
class Inner {
public List<Pojo> pojoList;
public List<Pojo> getPojoList() {
return pojoList;
}
#Override
public String toString() {
final StringBuilder sb = new StringBuilder("Inner{");
sb.append("pojoList=").append(pojoList);
sb.append('}');
return sb.toString();
}
}
class Pojo {
public Map<String, String> dataMap;
public int id;
#Override
public String toString() {
final StringBuilder sb = new StringBuilder("Pojo{");
sb.append("dataMap=").append(dataMap);
sb.append(", id=").append(id);
sb.append('}');
return sb.toString();
}
public Map<String, String> getDataMap() {
return dataMap;
}
public int getId() {
return id;
}
}
In my opinion Collections should never be null.
You could declare your pojoList and dataMap as private and instantiate them.
Your class then needs some add-methods. So you are sure getDataMap() never returns null:
class Pojo {
private Map<String, String> dataMap = new HashMap<>();
public Map<String, String> getDataMap() {
return dataMap;
}
public void add(String key, String value) {
dataMap.put(key, value);
}
}
Then you don't need to check for null:
Optional.ofNullable(testObj.inner)
.map(Inner::getPojoList)
.ifPresent(pojos -> pojos.forEach(pojo -> { pojo.getDataMap().clear(); }));

Resources