spring batch integration configuration with azure service bus - spring

I am trying to configure inbound and outbound adaptors as provided in the spring batch remote partitioning samples for Manager and worker beans. Having difficulty since they are configured in context of AMQPConnectionFactory.
However when I follow spring integration samples, there is no class which can provide Connection Factory. Help appreciated.
Below is sample code:-
import com.microsoft.azure.spring.integration.core.DefaultMessageHandler;
import com.microsoft.azure.spring.integration.core.api.CheckpointConfig;
import com.microsoft.azure.spring.integration.core.api.CheckpointMode;
import com.microsoft.azure.spring.integration.servicebus.inbound.ServiceBusQueueInboundChannelAdapter;
import com.microsoft.azure.spring.integration.servicebus.queue.ServiceBusQueueOperation;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.integration.partition.RemotePartitioningManagerStepBuilderFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.annotation.IntegrationComponentScan;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;
import org.springframework.util.concurrent.ListenableFutureCallback;
#Configuration
#IntegrationComponentScan
public class ManagerConfiguration {
private static final int GRID_SIZE = 3;
private static final String REQUEST_QUEUE_NAME = "digital.intg.batch.cm.request";
private static final String REPLY_QUEUE_NAME = "digital.intg.batch.cm.reply";
private static final String MANAGER_INPUT_CHANNEL = "manager.input";
private static final String MANGER_OUTPUT_CHANNEL = "manager.output";
private static final Log LOGGER = LogFactory.getLog(ManagerConfiguration.class);
private final JobBuilderFactory jobBuilderFactory;
private final RemotePartitioningManagerStepBuilderFactory managerStepBuilderFactory;
public ManagerConfiguration(JobBuilderFactory jobBuilderFactory,
RemotePartitioningManagerStepBuilderFactory managerStepBuilderFactory
) {
this.jobBuilderFactory = jobBuilderFactory;
this.managerStepBuilderFactory = managerStepBuilderFactory;
}
/*
* Configure outbound flow (requests going to workers)
*/
#Bean( name = MANGER_OUTPUT_CHANNEL )
public DirectChannel managerRequests() {
return new DirectChannel();
}
/*
* Configure inbound flow (replies coming from workers)
*/
#Bean( name = MANAGER_INPUT_CHANNEL )
public DirectChannel managerReplies() {
return new DirectChannel();
}
#Bean
public ServiceBusQueueInboundChannelAdapter managerQueueMessageChannelAdapter(
#Qualifier( MANAGER_INPUT_CHANNEL ) MessageChannel inputChannel, ServiceBusQueueOperation queueOperation) {
queueOperation.setCheckpointConfig(CheckpointConfig.builder().checkpointMode(CheckpointMode.MANUAL).build());
ServiceBusQueueInboundChannelAdapter adapter = new ServiceBusQueueInboundChannelAdapter(REPLY_QUEUE_NAME,
queueOperation);
adapter.setOutputChannel(inputChannel);
return adapter;
}
#Bean
#ServiceActivator( inputChannel = MANGER_OUTPUT_CHANNEL )
public MessageHandler managerQueueMessageSender(ServiceBusQueueOperation queueOperation) {
DefaultMessageHandler handler = new DefaultMessageHandler(REQUEST_QUEUE_NAME, queueOperation);
handler.setSendCallback(new ListenableFutureCallback<Void>() {
#Override
public void onSuccess(Void result) {
LOGGER.info("Manager Request Message was sent successfully.");
}
#Override
public void onFailure(Throwable ex) {
LOGGER.info("There was an error sending request message to worker.");
}
});
return handler;
}
#Bean
public IntegrationFlow managerOutboundFlow(MessageHandler managerQueueMessageSender) {
return IntegrationFlows
.from(managerRequests())
.handle(managerQueueMessageSender)
.get();
}
#Bean
public IntegrationFlow managerInboundFlow(ServiceBusQueueInboundChannelAdapter managerQueueMessageChannelAdapter) {
return IntegrationFlows
.from(managerQueueMessageChannelAdapter)
.channel(managerReplies())
.get();
}
/*
* Configure the manager step
*/
#Bean
public Step managerStep() {
return this.managerStepBuilderFactory.get("managerStep")
.partitioner("workerStep", new BasicPartitioner())
.gridSize(GRID_SIZE)
.outputChannel(managerRequests())
.inputChannel(managerReplies())
//.aggregator()
.build();
}
#Bean
public Job remotePartitioningJob() {
return this.jobBuilderFactory.get("remotePartitioningJob")
.start(managerStep())
.build();
}
}

The sample uses ActiveMQ because it is easily embeddable in a JVM for our tests and samples. But you can use any other broker that you want.
?? what should I inject here?
You should inject any dependency required by the queueMessageChannelAdapter handler:
.handle(queueMessageChannelAdapter)

Related

Spring rsocket security with Webflux security

My Application is a Spring Webflux application with Spring boot version 2.6.6.
Since, I have a chat and notification requirement for the logged in user, trying to use RSocket over websocket for notification & messaging along with Webflux for web based application.
Using Spring security for my web application with the config below and it is working. Now, not sure if I will be able to use the same security for RSocket as RSocket over websocket will be established when the user is logged in.
My Webflux security,
/**
*
*/
package com.TestApp.service.admin.spring.security;
import static java.util.stream.Collectors.toList;
import static org.springframework.security.config.Customizer.withDefaults;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.security.reactive.PathRequest;
import org.springframework.context.annotation.Bean;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.messaging.rsocket.RSocketStrategies;
import org.springframework.messaging.rsocket.annotation.support.RSocketMessageHandler;
import org.springframework.security.authentication.ReactiveAuthenticationManager;
import org.springframework.security.config.annotation.rsocket.EnableRSocketSecurity;
import org.springframework.security.config.annotation.rsocket.RSocketSecurity;
import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity;
import org.springframework.security.config.web.server.SecurityWebFiltersOrder;
import org.springframework.security.config.web.server.ServerHttpSecurity;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.ReactiveSecurityContextHolder;
import org.springframework.security.core.context.SecurityContext;
import org.springframework.security.messaging.handler.invocation.reactive.AuthenticationPrincipalArgumentResolver;
import org.springframework.security.rsocket.core.PayloadSocketAcceptorInterceptor;
import org.springframework.security.web.server.SecurityWebFilterChain;
import org.springframework.security.web.server.authentication.ServerAuthenticationFailureHandler;
import org.springframework.security.web.server.authentication.ServerAuthenticationSuccessHandler;
import org.springframework.security.web.server.authentication.logout.LogoutWebFilter;
import org.springframework.security.web.server.authentication.logout.RedirectServerLogoutSuccessHandler;
import org.springframework.security.web.server.authentication.logout.ServerLogoutHandler;
import org.springframework.security.web.server.authentication.logout.ServerLogoutSuccessHandler;
import org.springframework.security.web.server.authorization.HttpStatusServerAccessDeniedHandler;
import org.springframework.security.web.server.context.ServerSecurityContextRepository;
import org.springframework.security.web.server.csrf.CookieServerCsrfTokenRepository;
import org.springframework.security.web.server.util.matcher.AndServerWebExchangeMatcher;
import org.springframework.security.web.server.util.matcher.OrServerWebExchangeMatcher;
import org.springframework.security.web.server.util.matcher.PathPatternParserServerWebExchangeMatcher;
import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatcher;
import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatcher.MatchResult;
import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatchers;
import org.springframework.web.server.WebSession;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.testapp.service.admin.spring.TestAppProperties;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
#EnableWebFluxSecurity
public class AdminSecurityConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(AdminSecurityConfig.class);
private static final String[] DEFAULT_FILTER_MAPPING = new String[] { "/**" };
private static final String authenticateHeaderValue = "TestApp";
private static final String unauthorizedJsonBody = "{\"message\": \"You are not authorized\"}";
#Autowired
private TestAppProperties testAppProps;
#Bean
public SecurityWebFilterChain securitygWebFilterChain(final ServerHttpSecurity http,
final ReactiveAuthenticationManager authManager,
final ServerSecurityContextRepository securityContextRepository,
final TestAppAuthenticationFailureHandler failureHandler,
final ObjectProvider<TestAppLogoutHandler> availableLogoutHandlers) {
http.securityContextRepository(securityContextRepository);
return http.authorizeExchange().matchers(PathRequest.toStaticResources().atCommonLocations()).permitAll()
.pathMatchers(TestAppProps.getSecurity().getIgnorePatterns()).permitAll()
.anyExchange().authenticated().and().formLogin().loginPage(TestAppProps.getSecurity().getLoginPath())
.authenticationSuccessHandler(authSuccessHandler()).and().exceptionHandling()
.authenticationEntryPoint((exchange, exception) -> Mono.error(exception))
.accessDeniedHandler(new HttpStatusServerAccessDeniedHandler(HttpStatus.UNAUTHORIZED)).and().build();
}
#Bean
public ServerAuthenticationSuccessHandler authSuccessHandler() {
return new TestAppAuthSuccessHandler("/");
}
#Bean
public ServerLogoutSuccessHandler logoutSuccessHandler(String uri) {
RedirectServerLogoutSuccessHandler successHandler = new RedirectServerLogoutSuccessHandler();
successHandler.setLogoutSuccessUrl(URI.create(uri));
return successHandler;
}
#Bean(name = "failure-handler-bean")
public TestAppAuthenticationFailureHandler defaultFailureHandler() {
try {
new ObjectMapper().reader().readTree(unauthorizedJsonBody);
} catch (final IOException e) {
throw new IllegalArgumentException("'unauthorizedJsonBody' property is not valid JSON.", e);
}
return new TestAppAdminAuthFailureHandler(authenticateHeaderValue, unauthorizedJsonBody);
}
#Bean
public AuthenticatedPrinciplaProvider TestAppSecurityPrincipalProvider() {
return new TestAppSecurityContextPrincipleProvider();
}
}
public class TestAppSecurityContextPrincipleProvider implements AuthenticatedPrinciplaProvider {
#Override
public Mono<WhskrUserDetails> retrieveUser() {
return principalMono.flatMap(principal -> {
if (principal instanceof UsernamePasswordAuthenticationToken) {
final TestAppUserDetails user = (TestAppUserDetails) ((UsernamePasswordAuthenticationToken) principal)
.getPrincipal();
LOGGER.debug("User principal found for ID {} Org {} ", user.getUserId(), user.getOrgId());
return Mono.just(user);
}
return Mono.error(() -> new IllegalArgumentException(NO_USER_AUTH_ERROR));
})
}
}
This is working as expected. Have a login page and user gets redirected to the home page after the successful login.
Now, I am adding RSocket over websocket for messaging and notification for the logged in user.
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-security'
implementation 'org.springframework.security:spring-security-messaging'
implementation 'org.springframework.security:spring-security-rsocket'
implementation 'org.springframework.boot:spring-boot-starter-rsocket'
RSocketSecurityConfig,
#EnableWebFluxSecurity
public class AdminRSocketSecurityConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(AdminRSocketSecurityConfig.class);
private static final String[] DEFAULT_FILTER_MAPPING = new String[] { "/**" };
private static final String authenticateHeaderValue = "TestApp";
private static final String unauthorizedJsonBody = "{\"message\": \"You are not authorized\"}";
#Autowired
private TestAppProperties TestAppProps;
#Autowired
private AuthenticatedPrinciplaProvider secContext;
static final String RSOCKET_CONVERTER_BEAN_NAME = "RSocketAuthConverter";
private static final String HEADERS = "headers";
private static final MimeType COMPOSITE_METADATA_MIME_TYPE = MimeTypeUtils
.parseMimeType(WellKnownMimeType.MESSAGE_RSOCKET_COMPOSITE_METADATA.getString());
private static final MimeType APPLICATION_JSON_MIME_TYPE = MimeTypeUtils
.parseMimeType(WellKnownMimeType.APPLICATION_JSON.getString());
#Bean
public RSocketStrategies rsocketStrategies() {
return RSocketStrategies.builder()
.encoders(encoders -> encoders.add(new Jackson2CborEncoder()))
.decoders(decoders -> decoders.add(new Jackson2CborDecoder()))
.routeMatcher(new PathPatternRouteMatcher())
.build();
}
#Bean
public RSocketMessageHandler messageHandler(RSocketStrategies strategies) {
RSocketMessageHandler handler = new RSocketMessageHandler();
HandlerMethodArgumentResolver resolver = new AuthenticationPrincipalArgumentResolver();
handler.getArgumentResolverConfigurer().addCustomResolver(resolver);
handler.setRSocketStrategies(strategies);
return handler;
}
#Bean
public PayloadSocketAcceptorInterceptor authorization(final ReactiveAuthenticationManager authManager,
final RSocketSecurity security) {
security.authorizePayload(authorize -> authorize.setup().authenticated()).authenticationManager(authManager);
return security.build();
}
}
RSocketController,
#Controller
public class RSocketController {
private static final Logger LOGGER = LoggerFactory.getLogger(RSocketController.class);
private static final Map<Integer, Map<Integer, RSocketRequester>> CLIENT_REQUESTER_MAP = new HashMap<>();
static final String SERVER = "Server";
static final String RESPONSE = "Response";
static final String STREAM = "Stream";
static final String CHANNEL = "Channel";
#Autowired
private AuthenticatedPrinciplaProvider secContext;
#ConnectMapping
// void onConnect(RSocketRequester rSocketRequester, #Payload Integer userId) {
void onConnect(RSocketRequester rSocketRequester) {
secContext.retrieveUser().flatMap(usr -> {
LOGGER.info("Client connect request for userId {} ", usr.getUserId());
rSocketRequester.rsocket().onClose().doFirst(() -> {
CLIENT_REQUESTER_MAP.put(usr.getUserId(), rSocketRequester);
}).doOnError(error -> {
LOGGER.info("Client connect request for userId {} ", usr.getUserId());
}).doFinally(consumer -> {
LOGGER.info("Removing here for userId {} ", usr.getUserId());
if (CLIENT_REQUESTER_MAP.get(usr.getBranchId()) != null) {
CLIENT_REQUESTER_MAP.remove(usr.getUserId(), rSocketRequester);
}
}).subscribe();
return Mono.empty();
}).subscribe();
}
}
From the RSocket over WebSocket client, the call is not going to the controller as auth is failing.
But, When I set "authorize.setup().permitAll()" in my RSocketSecurityConfig authorization(), the call goes to the controller, but the retrieveUser() fails.
I am not sure, How can I use the same security which is being used for my web based application for RSocket security as well?
So, When user is not logged in to my web app, the rsocket over websocket should fail and it should work only when the user is logged in. The RSocket initial call is happening once the user is logged in.

Spring Integration how to use Control Bus with JavaConfig, no DSL

I'm having a few issues with Spring Integration and the control bus. I need to turn auto-start off on an InboundChannelAdapter. However when I do this I can't get the ControlBus to start the channel adapter.
I've searched for an answer online, but most of the examples use XML configuration.
Here is the entirety of my code:
package com.example.springintegrationdemo;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.integration.annotation.InboundChannelAdapter;
import org.springframework.integration.annotation.Poller;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.config.ExpressionControlBusFactoryBean;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.file.FileReadingMessageSource;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;
import org.springframework.messaging.support.GenericMessage;
import java.io.File;
#SpringBootApplication
#EnableIntegration
public class SpringIntegrationDemoApplication {
#Bean
public MessageChannel fileChannel() {
return new DirectChannel();
}
#Bean(name = "fileMessageSource")
#InboundChannelAdapter(channel = "fileChannel", poller = #Poller(fixedDelay = "1000"),autoStartup = "false")
public MessageSource<File> fileMessageSource() {
FileReadingMessageSource fileReadingMessageSource = new FileReadingMessageSource();
fileReadingMessageSource.setDirectory(new File("lz"));
return fileReadingMessageSource;
}
#Bean
#ServiceActivator(inputChannel = "fileChannel")
public MessageHandler messageHandler() {
MessageHandler messageHandler = message -> {
File f = (File) message.getPayload();
System.out.println(f.getAbsolutePath());
};
return messageHandler;
}
#Bean
MessageChannel controlChannel() {
return new DirectChannel();
}
#Bean
#ServiceActivator(inputChannel = "controlChannel")
ExpressionControlBusFactoryBean controlBus() {
ExpressionControlBusFactoryBean expressionControlBusFactoryBean = new ExpressionControlBusFactoryBean();
return expressionControlBusFactoryBean;
}
#Bean
CommandLineRunner commandLineRunner(#Qualifier("controlChannel") MessageChannel controlChannel) {
return (String[] args)-> {
System.out.println("Starting incoming file adapter: ");
boolean sent = controlChannel.send(new GenericMessage<>("#fileMessageSource.start()"));
System.out.println("Sent control message successfully? " + sent);
while(System.in.available() == 0) {
Thread.sleep(50);
}
};
}
public static void main(String[] args) {
SpringApplication.run(SpringIntegrationDemoApplication.class, args);
}
}
The message is sent to the control bus component successfully, but the inbound channel adapter never starts.
I would appreciate any help.
Thanks,
Dave
See here: https://docs.spring.io/spring-integration/docs/current/reference/html/configuration.html#annotations_on_beans
The fileMessageSource bean name is exactly for the FileReadingMessageSource. A SourcePollingChannelAdapter created from the InboundChannelAdapter has this bean name: springIntegrationDemoApplication.fileMessageSource.inboundChannelAdapter.
The #EndpointId can help you to simplify it.
In other words: everything is OK with your config, only the problem that you don't use the proper endpoint id to start the SourcePollingChannelAdapter.

Sync S3 Bucket and listen for changes

I've got an AWS S3 bucket, where I place on a weekly basis a new ZIP file.
I want to add a functionality to my existing Web Service, written with Spring Boot: synchronize the bucket locally and watch for changes.
For the time being, synchronization works well: whenever a new file is added to the bucket, it gets downloaded locally. However, I don't know to listen for file updates, this is, a method that fires when a new file is downloaded locally. Can it be done?
This is the piece of code I've:
# --------
# | AWS S3 |
# --------
s3.credentials-access-key=***
s3.credentials-secret-key=****
s3.bucket = my-bucket
s3.remote-dir = zips
s3.local-dir = D:/s3-bucket/
#Log4j2
#Configuration
public class S3Config {
public static final String OUT_CHANNEL_NAME = "s3filesChannel";
#Value("${s3.credentials-access-key}") private String accessKey;
#Value("${s3.credentials-secret-key}") private String secretKey;
#Value("${s3.remote-dir}") private String remoteDir;
#Value("${s3.bucket}") private String s3bucket;
#Value("${s3.local-dir}") private String localDir;
/*
* AWS S3
*/
#Bean
public AmazonS3 getAmazonS3(
){
BasicAWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 s3client = AmazonS3ClientBuilder
.standard()
.withRegion(Regions.EU_WEST_1)
.withCredentials(new AWSStaticCredentialsProvider(creds))
.build();
return s3client;
}
#Bean
public S3SessionFactory s3SessionFactory(AmazonS3 pAmazonS3) {
return new S3SessionFactory(pAmazonS3);
}
#Bean
public S3InboundFileSynchronizer s3InboundFileSynchronizer(S3SessionFactory pS3SessionFactory) {
S3InboundFileSynchronizer sync = new S3InboundFileSynchronizer(pS3SessionFactory);
sync.setPreserveTimestamp(true);
sync.setDeleteRemoteFiles(false);
String fullRemotePath = s3bucket.concat("/").concat(remoteDir);
sync.setRemoteDirectory(fullRemotePath);
sync.setFilter(new S3RegexPatternFileListFilter(".*\\.zip$"));
return sync;
}
#Bean
#InboundChannelAdapter(value = OUT_CHANNEL_NAME, poller = #Poller(fixedDelay = "30"))
public S3InboundFileSynchronizingMessageSource s3InboundFileSynchronizingMessageSource(
S3InboundFileSynchronizer pS3InboundFileSynchronizer
) {
S3InboundFileSynchronizingMessageSource messageSource = new S3InboundFileSynchronizingMessageSource(pS3InboundFileSynchronizer);
messageSource.setAutoCreateLocalDirectory(true);
messageSource.setLocalDirectory(new File(localDir));
messageSource.setLocalFilter(new AcceptOnceFileListFilter<File>());
return messageSource;
}
#Bean("s3filesChannel")
public PollableChannel s3FilesChannel() {
return new QueueChannel();
}
#Bean
public IntegrationFlow fileReadingFlow(
S3InboundFileSynchronizingMessageSource pS3InboundFileSynchronizingMessageSource,
GtfsBizkaibus pGtfsBizkaibus,
#Qualifier("fileProcessor") MessageHandler pMessageHandler) {
return IntegrationFlows
.from(pS3InboundFileSynchronizingMessageSource, e -> e.poller(p -> p.fixedDelay(5, TimeUnit.SECONDS)))
.handle(pMessageHandler)
.get();
}
#Bean("fileProcessor")
public MessageHandler fileProcessor() {
FileWritingMessageHandler handler = new FileWritingMessageHandler(new File(localDir));
handler.setExpectReply(false); // end of pipeline, reply not needed
handler.setFileExistsMode(FileExistsMode.APPEND);
handler.setNewFileCallback((file, msg) -> {
log.debug("New file created... " + file.getAbsolutePath());
});
return handler;
}
You can use S3 Event Notifications and an SQS queue. Basically when an object is added to your bucket, S3 can publish an event to a registered SQS queue. You can then have your on-premise application long poll the queue for new events and process any events that are added.
See here for more information.
Actually, the S3InboundFileSynchronizingMessageSource does all the necessary work for you: when new file is added into a remote bucket, it is downloaded to local dir and produced as a payload in the message to be sent to the configured channel.
When remote file is modified, it is also downloaded to local dir.
Starting with version 5.0, the AbstractInboundFileSynchronizingMessageSource provides this option:
/**
* Switch the local {#link FileReadingMessageSource} to use its internal
* {#code FileReadingMessageSource.WatchServiceDirectoryScanner}.
* #param useWatchService the {#code boolean} flag to switch to
* {#code FileReadingMessageSource.WatchServiceDirectoryScanner} on {#code true}.
* #since 5.0
*/
public void setUseWatchService(boolean useWatchService) {
this.fileSource.setUseWatchService(useWatchService);
if (useWatchService) {
this.fileSource.setWatchEvents(
FileReadingMessageSource.WatchEventType.CREATE,
FileReadingMessageSource.WatchEventType.MODIFY,
FileReadingMessageSource.WatchEventType.DELETE);
}
}
If that makes some sense to you.
But yeah... with S3 to SQS notification it is also going to be a good solution. There is an SqsMessageDrivenChannelAdapter in Spring Integration AWS project.
Finally, as #Artem Bilian suggested, I've make use of #ServiceActivator annotation. Here it's the full example:
import java.io.File;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.annotation.InboundChannelAdapter;
import org.springframework.integration.annotation.Poller;
import org.springframework.integration.annotation.ServiceActivator;
import org.springframework.integration.aws.inbound.S3InboundFileSynchronizer;
import org.springframework.integration.aws.inbound.S3InboundFileSynchronizingMessageSource;
import org.springframework.integration.aws.support.S3SessionFactory;
import org.springframework.integration.aws.support.filters.S3RegexPatternFileListFilter;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.file.FileWritingMessageHandler;
import org.springframework.integration.file.support.FileExistsMode;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageHandler;
import org.springframework.messaging.PollableChannel;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import lombok.extern.log4j.Log4j2;
#Log4j2
#Configuration
public class S3Config {
public static final String IN_CHANNEL_NAME = "s3filesChannel";
#Value("${s3.credentials-access-key}") private String accessKey;
#Value("${s3.credentials-secret-key}") private String secretKey;
#Value("${s3.remote-dir}") private String remoteDir;
#Value("${s3.bucket}") private String s3bucket;
#Value("${s3.local-dir}") private String localDir;
/*
* AWS S3
*/
#Bean
public AmazonS3 getAmazonS3(
){
BasicAWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 s3client = AmazonS3ClientBuilder
.standard()
.withRegion(Regions.EU_WEST_1)
.withCredentials(new AWSStaticCredentialsProvider(creds))
.build();
return s3client;
}
#Bean
public S3SessionFactory s3SessionFactory(AmazonS3 pAmazonS3) {
return new S3SessionFactory(pAmazonS3);
}
#Bean
public S3InboundFileSynchronizer s3InboundFileSynchronizer(S3SessionFactory pS3SessionFactory) {
S3InboundFileSynchronizer sync = new S3InboundFileSynchronizer(pS3SessionFactory);
sync.setPreserveTimestamp(true);
sync.setDeleteRemoteFiles(false);
String fullRemotePath = s3bucket.concat("/").concat(remoteDir);
sync.setRemoteDirectory(fullRemotePath);
sync.setFilter(new S3RegexPatternFileListFilter(".*\\.zip$"));
return sync;
}
#Bean
#InboundChannelAdapter(value = IN_CHANNEL_NAME, poller = #Poller(fixedDelay = "30"))
public S3InboundFileSynchronizingMessageSource s3InboundFileSynchronizingMessageSource(
S3InboundFileSynchronizer pS3InboundFileSynchronizer
) {
S3InboundFileSynchronizingMessageSource messageSource = new S3InboundFileSynchronizingMessageSource(pS3InboundFileSynchronizer);
messageSource.setAutoCreateLocalDirectory(true);
messageSource.setLocalDirectory(new File(localDir));
messageSource.setUseWatchService(true);
return messageSource;
}
#Bean("s3filesChannel")
public PollableChannel s3FilesChannel() {
return new QueueChannel();
}
#Bean
public IntegrationFlow fileReadingFlow(
S3InboundFileSynchronizingMessageSource pS3InboundFileSynchronizingMessageSource,
#Qualifier("fileProcessor") MessageHandler pMessageHandler) {
return IntegrationFlows
.from(pS3InboundFileSynchronizingMessageSource, e -> e.poller(p -> p.fixedDelay(5, TimeUnit.SECONDS)))
.handle(pMessageHandler)
.get();
}
#Bean("fileProcessor")
public MessageHandler fileProcessor() {
FileWritingMessageHandler handler = new FileWritingMessageHandler(new File(localDir));
handler.setExpectReply(false); // end of pipeline, reply not needed
handler.setFileExistsMode(FileExistsMode.REPLACE);
return handler;
}
#ServiceActivator(inputChannel = IN_CHANNEL_NAME, poller = #Poller(fixedDelay = "30"))
public void asada(Message<?> message) {
// TODO
log.debug("<< New message!");
}
}
Note that I've replaced OUT_CHANNEL_NAME by IN_CHANNEL_NAME.
PS: I'm almost new to Spring Integration, so I'm still learning its concepts.

How to wait for a spring jms listener thread to finish executing in Junit test

I have a spring boot application that uses spring-JMS. Is there any way to tell the test method to wait the jms lister util it finishes executing without using latches in the actual code that will be tested?
Here is the JMS listener code:
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.stereotype.Component;
import javax.jms.Message;
import javax.jms.QueueSession;
#Component
public class MyListener {
#Autowired
MyProcessor myProcessor;
#JmsListener(destination = "myQueue", concurrency = "1-4")
private void onMessage(Message message, QueueSession session) {
myProcessor.processMessage(message, session);
}
}
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.jms.Message;
import javax.jms.QueueSession;
#Component
public class MyProcessor {
public void processMessage(Message msg, QueueSession session) {
//Here I have some code.
}
}
import org.apache.activemq.command.ActiveMQTextMessage;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.QueueSession;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
#SpringBootTest
#ExtendWith(SpringExtension.class)
#ActiveProfiles("test")
public class IntegrationTest {
#Autowired
private JmsTemplate JmsTemplate;
#Test
public void myTest() throws JMSException {
Message message = new ActiveMQTextMessage();
jmsTemplate.send("myQueue", session -> message);
/*
Here I have some testing code. How can I tell the application
to not execute this testing code until all JMS lister threads
finish executing.
*/
}
}
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.broker.BrokerService;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.jms.annotation.EnableJms;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.util.SocketUtils;
import javax.jms.ConnectionFactory;
#EnableJms
#Configuration
#Profile("test")
public class JmsTestConfig {
public static final String BROKER_URL =
"tcp://localhost:" + SocketUtils.findAvailableTcpPort();
#Bean
public BrokerService brokerService() throws Exception {
BrokerService brokerService = new BrokerService();
brokerService.setPersistent(false);
brokerService.addConnector(BROKER_URL);
return brokerService;
}
#Bean
public ConnectionFactory connectionFactory() {
return new ActiveMQConnectionFactory(BROKER_URL);
}
#Bean
public JmsTemplate jmsTemplate(ConnectionFactory connectionFactory) {
JmsTemplate jmsTemplate = new JmsTemplate(connectionFactory);
return jmsTemplate;
}
}
Note: Is it applicable to solve this without adding testing purpose code to the implementation code (MyListener and MyProcessor).
Proxy the listener and add an advice to count down a latch; here's one I did for a KafkaListener recently...
#Test
public void test() throws Exception {
this.template.send("so50214261", "foo");
assertThat(TestConfig.latch.await(10, TimeUnit.SECONDS)).isTrue();
assertThat(TestConfig.received.get()).isEqualTo("foo");
}
#Configuration
public static class TestConfig {
private static final AtomicReference<String> received = new AtomicReference<>();
private static final CountDownLatch latch = new CountDownLatch(1);
#Bean
public static MethodInterceptor interceptor() {
return invocation -> {
received.set((String) invocation.getArguments()[0]);
return invocation.proceed();
};
}
#Bean
public static BeanPostProcessor listenerAdvisor() {
return new ListenerWrapper(interceptor());
}
}
public static class ListenerWrapper implements BeanPostProcessor, Ordered {
private final MethodInterceptor interceptor;
#Override
public int getOrder() {
return Ordered.HIGHEST_PRECEDENCE;
}
public ListenerWrapper(MethodInterceptor interceptor) {
this.interceptor = interceptor;
}
#Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
if (bean instanceof Listener) {
ProxyFactory pf = new ProxyFactory(bean);
NameMatchMethodPointcutAdvisor advisor = new NameMatchMethodPointcutAdvisor(this.interceptor);
advisor.addMethodName("listen");
pf.addAdvisor(advisor);
return pf.getProxy();
}
return bean;
}
}
(but you should move the countDown to after the invocation proceed()).
A method annotated with #JmsListener deletes the message after it finishes, so a good option is to read the queue for existing messages and assume the queue is empty after your method is done. Here is the piece of code for counting the messages from the queue.
private int countMessages() {
return jmsTemplate.browse(queueName, new BrowserCallback<Integer>() {
#Override
public Integer doInJms(Session session, QueueBrowser browser) throws JMSException {
return Collections.list(browser.getEnumeration()).size();
}
});
}
Following is the code for testing the countMessages() method.
jmsTemplate.convertAndSend(queueName, "***MESSAGE CONTENT***");
while (countMessages() > 0) {
log.info("number of pending messages: " + countMessages());
Thread.sleep(1_000l);
}
// continue with your logic here
I've based my solution on the answer given by Gary Russell, but rather put the CountDownLatch in an Aspect, using Spring AOP (or the spring-boot-starter-aop variant).
public class TestJMSConfiguration {
private static final Logger LOGGER = LoggerFactory.getLogger(TestJMSConfiguration.class);
public static final CountDownLatch countDownLatch = new CountDownLatch(1);
#Component
#Aspect
public static class LatchCounterAspect {
#Pointcut("execution(public void be.infrabel.rocstdm.application.ROCSTDMMessageListener.onMessage(javax.jms.TextMessage))")
public void onMessageMethod() {};
#After(value = "onMessageMethod()")
public void countDownLatch() {
countDownLatch.countDown();
LOGGER.info("CountDownLatch called. Count now at: {}", countDownLatch.getCount());
}
}
A snippet of the test:
JmsTemplate jmsTemplate = new JmsTemplate(this.embeddedBrokerConnectionFactory);
jmsTemplate.convertAndSend("AQ.SOMEQUEUE.R", message);
TestJMSConfiguration.countDownLatch.await();
verify(this.listenerSpy).putResponseOnTargetQueueAlias(messageCaptor.capture());
RouteMessage outputMessage = messageCaptor.getValue();
The listenerSpy is a #SpyBean annotated field of the type of my MessageListener. The messageCaptor is a field of type ArgumentCaptor<MyMessageType> annotated with #Captor. Both of these are coming from mockito so you need to run/extend your test with both MockitoExtension (or -Runner) along with the SpringExtension (or -Runner).
My code puts an object on an outbound queue after processing the incoming message, hence the putResponseOnTargetQueueAlias method. The captor is to intercept that object and do my assertions accordingly. The same strategy could be applied to capture some other object in your logic.

Method with #JmsListener not executed when setting custom message converter

Hy,
Having these three classes I have found a problem:
package xpadro.spring.jms;
import javax.jms.ConnectionFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.support.converter.MarshallingMessageConverter;
import org.springframework.oxm.jaxb.Jaxb2Marshaller;
#SpringBootApplication
public class JmsJavaconfigApplication {
public static void main(String[] args) {
SpringApplication.run(JmsJavaconfigApplication.class, args);
}
#Bean
public Jaxb2Marshaller jaxb2Marshaller(){
Jaxb2Marshaller jaxb = new Jaxb2Marshaller();
Class array[] = new Class[1];
array[0]= xpadro.spring.jms.model.Order.class;
jaxb.setClassesToBeBound(array);
return jaxb;
}
#Bean
#Autowired
public MarshallingMessageConverter marshallingMessageConverter( Jaxb2Marshaller marshaller){
MarshallingMessageConverter mc = new MarshallingMessageConverter();
mc.setMarshaller(marshaller);
mc.setUnmarshaller(marshaller);
return mc;
}
#Bean
#Autowired
public JmsTemplate init(MarshallingMessageConverter messageConverter,ConnectionFactory connectionFactory){
JmsTemplate template = new JmsTemplate();
template.setMessageConverter(messageConverter);
template.setConnectionFactory(connectionFactory);
return template;
}
}
Next, the model:
package xpadro.spring.jms.model;
import java.io.Serializable;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
#XmlRootElement(name = "order")
#XmlAccessorType(XmlAccessType.FIELD)
public class Order implements Serializable {
private static final long serialVersionUID = -797586847427389162L;
#XmlElement(required = true)
private String id="";
public Order() {
}
public Order(String id) {
this.id = id;
}
public String getId() {
return id;
}
}
Next, the listener:
package xpadro.spring.jms.listener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.stereotype.Component;
import xpadro.spring.jms.model.Order;
import xpadro.spring.jms.service.StoreService;
#Component
public class SimpleListener {
private final StoreService storeService;
#Autowired
public SimpleListener(StoreService storeService) {
this.storeService = storeService;
}
#JmsListener(destination = "simple.queue")
public void receiveOrder(Order order) {
storeService.registerOrder(order);
}
}
The class that sends messages:
package xpadro.spring.jms.service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.stereotype.Service;
import xpadro.spring.jms.model.Order;
#Service
public class ClientServiceImpl implements ClientService {
private static final String SIMPLE_QUEUE = "simple.queue";
private static final String IN_QUEUE = "in.queue";
private final JmsTemplate jmsTemplate;
#Autowired
public ClientServiceImpl(JmsTemplate jmsTemplate) {
this.jmsTemplate = jmsTemplate;
}
#Override
public void addOrder(Order order) {
//MessageRegistered with a bean
jmsTemplate.convertAndSend(SIMPLE_QUEUE, order);
}
#Override
public void registerOrder(Order order) {
//MessageRegistered with a bean
jmsTemplate.convertAndSend(IN_QUEUE, order);
}
}
The point is that listener(method receiveOrder()) is not executed when I set a custom message converter in the class JmsJavaconfigApplication. When I dont set it, spring boot sets a SimpleMessageConverter and the listener is executed.
Any wrong or missed configuration?
Thanks
You need to provide an marshalling converter to the #JmsListener to unmarshall the message, by configuring the listener container factory see the documentation.

Resources