PooledCloseableHttpClient impacting performance throughput - performance

Question

I have been using Spring-Integration to call a REST api, however the http-client which comes by default with Spring-Integration does not support connection-pooling or reusability, so I customized to use PoolingHttpClientConnectionManager
But now Spring-integration stopped picking up the JKS files in my classpath, so I built my own SSL Context, however building this SSL Context caused significant drop in performance
For 100 concurrent threads,
Using http client I got 200 TPS
Using PoolingHttpClientConnectionManager and SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER, I got up to 380 TPS.
Building SSL context from JKS buildSslContext() I get less than 30 TPS :(
Context.xml
<int:gateway id="ServiceRequestGateway"
service-interface="com.company.security.integration.RequestGateway"
default-request-channel="RequestChannel"
default-reply-channel="ResponseChannel">
<int:default-header name="Accept" value="application/json; v=5"/>
<int:default-header name="Content-Type" value="application/json; v=5"/>
<int:default-header name="ServiceType" expression="#args[1]"/>
</int:gateway>
<int-http:outbound-gateway
id="Outbound_Gateway"
request-channel="RequestChannel"
reply-channel="ResponseChannel"
request-factory="requestFactory"
header-mapper="headerMapper"
url="${service.host}/{xyzServiceType}"
http-method="POST"
expected-response-type="java.lang.String"
extract-request-payload="true">
<int-http:uri-variable name="ServiceType" expression="headers['xyzServiceType']" />
</int-http:outbound-gateway>
<!--Connection Pooling/Keep Alive/Retry-->
<bean id="httpClient" class="com.capitalone.security.config.PooledCloseableHttpClient">
</bean>
<bean id="requestFactory"
class="org.springframework.http.client.HttpComponentsClientHttpRequestFactory">
<constructor-arg ref="httpClient"/>
<property name="connectTimeout" value="5000"/>
<property name="readTimeout" value="5000"/>
</bean>
PooledCloseableHttpClient
public class PooledCloseableHttpClient implements FactoryBean {
#Autowired
S3ClientUtil s3Client;
// For TLS/SSL connectivity from this client to service
#Value("${jks.filename}")
String jksFile;
// Password for Java keystores
#Value("${keystore.password}")
String keystorePassword;
private int maxRetries = 2;
//1 second
#Value("${rest.call.request.retryInterval:1000}")
private int retryInterval = 1000;
#Value("${rest.call.request.keepAliveTime:60}")
private int keepAliveTime = 60;
#Value("${rest.call.request.maxConnection:200}")
private int maxConnection = 200;
#Value("${rest.call.request.maxConnectionsPerRoute:100}")
private int maxConnectionsPerRoute = 100 ;
SSLConnectionSocketFactory sslConnectionSocketFactory;
// Custom Keep-Alive
ConnectionKeepAliveStrategy keepAliveStrategy = (response, context) -> {
HeaderElementIterator it = new BasicHeaderElementIterator
(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase
("timeout")) {
return Long.parseLong(value) * 1000;
}
}
return keepAliveTime * 1000;
};
// Called once during initialization to get JKS file from Cloud
private SSLContext buildSslContext() {
try {
// Get the JKS contents and then use the pooling connection manager below
File keyStoreFile = s3Client.importKeystoreFile(jksFile);
// Build key store from JKS file downloaded from S3
final KeyStore keyStore = KeyStore.getInstance("JKS");
InputStream is = null;
try {
is = new FileInputStream(keyStoreFile); // Get Keystore
keyStore.load(is, keystorePassword.toCharArray()); //Get keystore password
} finally {
IOUtils.closeQuietly(is);
}
// Build SSL Context
SSLContextBuilder sslBuilder = new SSLContextBuilder();
sslBuilder.loadKeyMaterial(keyStore, keystorePassword.toCharArray());
sslBuilder.loadTrustMaterial(keyStoreFile, keystorePassword.toCharArray());
return sslBuilder.build();
} catch (final GeneralSecurityException | IOException exc) {
return null;
}
}
#Override
public Object getObject() throws Exception {
//Build PoolingHttpClientConnectionManager
PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(
RegistryBuilder.<ConnectionSocketFactory> create()
.register("https", new SSLConnectionSocketFactory(buildSslContext(), new NoopHostnameVerifier()))
.register("http", new PlainConnectionSocketFactory()).build());
// Build HttpClient
HttpClientBuilder httpClientBuilder = HttpClients.custom().useSystemProperties().setConnectionManager(poolingConnectionManager)
.setKeepAliveStrategy(keepAliveStrategy)
.setSSLSocketFactory(sslConnectionSocketFactory)
.setServiceUnavailableRetryStrategy(new ServiceUnavailableRetryStrategy(maxRetries, retryInterval));
return httpClientBuilder.build();
}
#Override
public Class<?> getObjectType() {
return CloseableHttpClient.class;
}
#Override
public boolean isSingleton() {
return true;
}
}

Answer 1

Here is the refactored HttpClient class which gave me the optimum performance.
public class PooledCloseableHttpClient implements FactoryBean {
#Autowired
S3ClientUtil s3Client;
// For TLS/SSL connectivity from this client to service
#Value("${jks.filename}")
String jksFile;
// Password for Java keystores
#Value("${keystore.password}")
String keystorePassword;
private int maxRetries = 2;
//1 second
#Value("${rest.call.request.retryInterval:1000}")
private int retryInterval = 1000;
#Value("${rest.call.request.keepAliveTime:60}")
private int keepAliveTime = 60;
#Value("${rest.call.request.maxConnection:200}")
private int maxConnection = 200;
#Value("${rest.call.request.maxConnectionsPerRoute:100}")
private int maxConnectionsPerRoute = 100 ;
SSLConnectionSocketFactory sslConnectionSocketFactory;
// Custom Keep-Alive
ConnectionKeepAliveStrategy keepAliveStrategy = (response, context) -> {
HeaderElementIterator it = new BasicHeaderElementIterator
(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase
("timeout")) {
return Long.parseLong(value) * 1000;
}
}
return keepAliveTime * 1000;
};
// Called once during initialization to get JKS file from Cloud
private SSLContext buildSslContext() {
try {
// Get the JKS contents and then use the pooling connection manager below
File keyStoreFile = s3Client.importKeystoreFile(jksFile);
// Build key store from JKS file downloaded from S3
final KeyStore keyStore = KeyStore.getInstance("JKS");
InputStream is = null;
try {
is = new FileInputStream(keyStoreFile); // Get Keystore
keyStore.load(is, keystorePassword.toCharArray()); //Get keystore password
} finally {
IOUtils.closeQuietly(is);
}
// Build SSL Context
SSLContextBuilder sslBuilder = new SSLContextBuilder();
sslBuilder.loadKeyMaterial(keyStore, keystorePassword.toCharArray());
sslBuilder.loadTrustMaterial(keyStoreFile, keystorePassword.toCharArray());
return sslBuilder.build();
} catch (final GeneralSecurityException | IOException exc) {
return null;
}
}
#Override
public Object getObject() throws Exception {
//Build PoolingHttpClientConnectionManager
PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager(
RegistryBuilder.<ConnectionSocketFactory> create()
.register("https", new SSLConnectionSocketFactory(buildSslContext(), new NoopHostnameVerifier()))
.register("http", new PlainConnectionSocketFactory()).build())
poolingConnectionManager.setMaxTotal(maxConnection);
poolingConnectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
// Build HttpClient
HttpClientBuilder httpClientBuilder = HttpClients.custom().useSystemProperties().setConnectionManager(poolingConnectionManager)
.setKeepAliveStrategy(keepAliveStrategy)
.setSSLSocketFactory(sslConnectionSocketFactory)
.setConnectionReuseStrategy((arg0, arg1) -> true)
.setMaxConnTotal(maxConnection)
.setMaxConnPerRoute(maxConnectionsPerRoute)
.setServiceUnavailableRetryStrategy(new ServiceUnavailableRetryStrategy(maxRetries, retryInterval));
return httpClientBuilder.build();
}
#Override
public Class<?> getObjectType() {
return CloseableHttpClient.class;
}
#Override
public boolean isSingleton() {
return true;
}
}