Cannot Create Database connection in Spring-Boot - spring

Question

I am deploying a spring boot application on Openshift Tomcat 7.0.54 environment. We have total four instances of Tomcat, and when I deploy the application, it gets deploy into every instance of tomcat.
Here is the problem, I am only able to open database connection for one instance for rest of the three instances after some time I get the exception (shown below).
I am not able to find the root cause for not getting the connection. I am using SpringJDBC in the project (not using hibernate).
Since I have to move the code from Development->Stage->Production, I cannot hard code the database login credentials because of that I am using JDNI for getting it from the environment variable based on the environment in which application is running.
Application.java
import javax.naming.NamingException;
import javax.sql.DataSource;
import org.apache.catalina.Context;
import org.apache.catalina.deploy.ContextResource;
import org.apache.catalina.startup.Tomcat;
import org.apache.log4j.Logger;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.web.ErrorMvcAutoConfiguration;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.context.embedded.EmbeddedServletContainerFactory;
import org.springframework.boot.context.embedded.tomcat.TomcatEmbeddedServletContainer;
import org.springframework.boot.context.embedded.tomcat.TomcatEmbeddedServletContainerFactory;
import org.springframework.boot.web.support.SpringBootServletInitializer;
import org.springframework.context.annotation.Bean;
import org.springframework.jndi.JndiObjectFactoryBean;
#SpringBootApplication
#EnableAutoConfiguration(exclude = { ErrorMvcAutoConfiguration.class })
public class Application extends SpringBootServletInitializer {
private static final Logger LOGGER = Logger.getLogger(Application.class);
public static void main(String[] args) {
LOGGER.info("Spring Boot Application Started");
SpringApplication.run(Application.class, args);
LOGGER.info("Spring Boot Application Ended");
}
#Override
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
return application.sources(Application.class);
}
#Bean
public EmbeddedServletContainerFactory tomcatFactory() {
return new TomcatEmbeddedServletContainerFactory() {
#Override
protected TomcatEmbeddedServletContainer getTomcatEmbeddedServletContainer(Tomcat tomcat) {
tomcat.enableNaming();
return super.getTomcatEmbeddedServletContainer(tomcat);
}
#Override
protected void postProcessContext(Context context) {
ContextResource resource = new ContextResource();
resource.setName("jdbc/grs_lae_cust_db");
resource.setType(DataSource.class.getName());
resource.setProperty("driverClassName", "oracle.jdbc.OracleDriver");
resource.setProperty("url","jdbc:oracle:thin:#(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno))(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno)))(CONNECT_DATA=(SERVICE_NAME=service.company.com)(SERVER=DEDICATED)))");
resource.setAuth("Container");
resource.setProperty("username", "username");
resource.setProperty("password", "password");
context.getNamingResources().addResource(resource);
}
};
}
#Bean(destroyMethod = "")
public DataSource jndiDataSource() throws IllegalArgumentException, NamingException {
LOGGER.debug("Entering JNDIDataSource");
JndiObjectFactoryBean bean = new JndiObjectFactoryBean();
bean.setJndiName("java:comp/env/jdbc/grs_lae_cust_db");
bean.setProxyInterface(DataSource.class);
bean.setLookupOnStartup(false);
bean.afterPropertiesSet();
LOGGER.info("Leaving JNDIDataSource");
return (DataSource) bean.getObject();
}
}
context.xml
<?xml version='1.0' encoding='utf-8'?>
<Context>
<!-- Default set of monitored resources -->
<WatchedResource>WEB-INF/web.xml</WatchedResource>
<Resource name="jdbc/grs_lae_cust_db"
auth="Container"
type="javax.sql.DataSource"
driverClassName="oracle.jdbc.OracleDriver"
url="${OPENSHIFT_DB_URL}"
username="${OPENSHIFT_DB_USERNAME}"
password="${OPENSHIFT_DB_PASSWORD}"
maxActive="150"
maxIdle="10"
maxWait="30000"
initialSize="2"
minIdle = "2"
jmxEnabled="true"
timeBetweenEvictionRunsMillis="180000"
minEvictableIdleTimeMillis="300000"
logAbandoned="true"
testOnBorrow="true"
testOnConnect="false"
suspectTimeout="120"
validationQuery="select 1 from dual"
validationInterval="200"
initSQL="select 1 from dual"
removeAbandoned="false"
removeAbandonedTimeout="1800"
maxAge ="1800000"
accessToUnderlyingConnectionAllowed="true"/>
</Context>
pre_restart_jbossews and pre_start_jbossews
###########START OF SCRIPT################### #
#!/bin/bash
export ORACLE_HOME="/opt/oracle/product/current"
export LD_LIBRARY_PATH="/opt/oracle/product/current/lib"
export CLASSPATH="/opt/oracle/product/current/jdbc/lib/ojdbc6.jar"
appname=`echo $OPENSHIFT_APP_NAME`
case "$appname" in
*dev) export JAVA_OPTS_EXT="${JAVA_OPTS_EXT} -D.life=dev"
export OPENSHIFT_company_LDAP="ldap://dsxstage.company.com:389"
export OPENSHIFT_DB_URL="jdbc:oracle:thin:#(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=ocm-i-nprd)(PORT=1600))(CONNECT_DATA=(SERVICE_NAME=MKWBDEV.company.com)))"
export OPENSHIFT_DB_USERNAME="username"
export OPENSHIFT_DB_PASSWORD='password';;
*stage|*stg) export JAVA_OPTS_EXT="${JAVA_OPTS_EXT} -D.life=stage"
export OPENSHIFT_company_LDAP="ldap://dsx.company.com:389"
export OPENSHIFT_DB_URL="jdbc:oracle:thin:#(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno))(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno)))(CONNECT_DATA=(SERVICE_NAME=servicenema.company.com)(SERVER=DEDICATED)))"
export OPENSHIFT_DB_USERNAME="username"
export OPENSHIFT_DB_PASSWORD='password';;
*lt) export JAVA_OPTS_EXT="${JAVA_OPTS_EXT} -D.life=lt"
export OPENSHIFT_company_LDAP="ldap://dsx.company.com:389"
export OPENSHIFT_DB_URL="jdbc:oracle:thin:#(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno))(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno)))(CONNECT_DATA=(SERVICE_NAME=servicenema.company.com)(SERVER=DEDICATED)))"
export OPENSHIFT_DB_USERNAME="username"
export OPENSHIFT_DB_PASSWORD='password';;
*prod|*prd) export JAVA_OPTS_EXT="${JAVA_OPTS_EXT} -D.life=prod"
export OPENSHIFT_company_LDAP="ldap://dsx.company.com:389"
export OPENSHIFT_DB_URL="jdbc:oracle:thin:#(DESCRIPTION=(ADDRESS_LIST=(LOAD_BALANCE=ON)(FAILOVER=ON)(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno))(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno)))(CONNECT_DATA=(SERVICE_NAME=servicenema.company.com)(SERVER=DEDICATED)))"
export OPENSHIFT_DB_USERNAME="username"
export OPENSHIFT_DB_PASSWORD='password';;
*) export JAVA_OPTS_EXT="${JAVA_OPTS_EXT} -D.life=dev"
export OPENSHIFT_company_LDAP="ldap://dsxstage.company.com:389"
export OPENSHIFT_DB_URL="jdbc:oracle:thin:#(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=IP)(PORT=portno))(CONNECT_DATA=(SERVICE_NAME=servicename.company.com)))"
export OPENSHIFT_DB_USERNAME="username"
export OPENSHIFT_DB_PASSWORD='password';;
esac
###############END OF SCRIPT##################
assembly.xml
<?xml version="1.0"?>
<assembly>
<id>${project.version}</id>
<formats>
<format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>${project.build.directory}/../package</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
</fileSets>
</assembly>
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>tomcat7</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>cctgmap-tomcat7</name>
<description>Eloqua integration</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.4.1.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<tomcat.version>7.0.59</tomcat.version>
<java.version>1.7</java.version>
<guava.version>12.0</guava.version>
<json-simple.version>1.1.1</json-simple.version>
<commons-codec.verison>1.10</commons-codec.verison>
<log4j-core.version>2.4.1</log4j-core.version>
<log4j-api.version>2.4.1</log4j-api.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.session</groupId>
<artifactId>spring-session</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.session</groupId>
<artifactId>spring-session-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-freemarker</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jersey</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-mail</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<!-- <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency> -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc-portlet</artifactId>
</dependency>
<!-- http://mvnrepository.com/artifact/commons-codec/commons-codec -->
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons-codec.verison}</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>${json-simple.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j-core.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j-api.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<!-- Dependencies added for WAS and Oracle -->
<dependency>
<groupId>com.oracle</groupId>
<artifactId>classes12</artifactId>
<version>10.2.0.5</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.oracle</groupId>
<artifactId>ojdbc6</artifactId>
<version>11.2.0.3</version>
<scope>provided</scope>
</dependency>
<!-- <dependency>
<groupId>com.oracle</groupId>
<artifactId>ojdbc14</artifactId>
<version>10.2.0.5</version>
</dependency> -->
<!-- Dependencies WAS and Oracle ends -->
<!-- Server Dependency -->
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-juli</artifactId>
<version>${tomcat.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-dbcp</artifactId>
<version>7.0.30</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jdbc</artifactId>
</dependency>
<!-- Server Dependency ends -->
</dependencies>
<build>
<finalName>cctgmap</finalName>
<plugins>
<plugin>
<groupId>org.apache.tomcat.maven</groupId>
<artifactId>tomcat7-maven-plugin</artifactId>
<version>2.2</version>
<configuration>
<server>TomcatServer</server>
<port>8008</port>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<version>2.1.1</version>
<configuration>
<outputDirectory>${project.build.directory}/../package/dependencies/jbossews/webapps </outputDirectory>
<warName>ROOT</warName>
</configuration>
</plugin>
</plugins>
</build>
</project>
I am getting this error
Feb 25, 2017 1:17:48 AM org.apache.naming.NamingContext lookup
WARNING: Unexpected exception resolving reference
org.apache.tomcat.dbcp.dbcp.SQLNestedException: Cannot create PoolableConnectionFactory (IO Error: The Network Adapter could not establish the connection)
at org.apache.tomcat.dbcp.dbcp.BasicDataSource.createPoolableConnectionFactory(BasicDataSource.java:1551)
at org.apache.tomcat.dbcp.dbcp.BasicDataSource.createDataSource(BasicDataSource.java:1390)
at org.apache.tomcat.dbcp.dbcp.BasicDataSource.getLogWriter(BasicDataSource.java:1100)
at org.apache.tomcat.dbcp.dbcp.BasicDataSourceFactory.createDataSource(BasicDataSourceFactory.java:350)
at org.apache.tomcat.dbcp.dbcp.BasicDataSourceFactory.getObjectInstance(BasicDataSourceFactory.java:156)
at org.apache.naming.factory.ResourceFactory.getObjectInstance(ResourceFactory.java:141)
at javax.naming.spi.NamingManager.getObjectInstance(NamingManager.java:321)
at org.apache.naming.NamingContext.lookup(NamingContext.java:842)
at org.apache.naming.NamingContext.lookup(NamingContext.java:153)
at org.apache.naming.NamingContext.lookup(NamingContext.java:830)
at org.apache.naming.NamingContext.lookup(NamingContext.java:153)
at org.apache.naming.NamingContext.lookup(NamingContext.java:830)
at org.apache.naming.NamingContext.lookup(NamingContext.java:153)
at org.apache.naming.NamingContext.lookup(NamingContext.java:830)
at org.apache.naming.NamingContext.lookup(NamingContext.java:167)
at org.apache.naming.SelectorContext.lookup(SelectorContext.java:156)
at javax.naming.InitialContext.lookup(InitialContext.java:411)
at org.springframework.jndi.JndiTemplate$1.doInContext(JndiTemplate.java:155)
at org.springframework.jndi.JndiTemplate.execute(JndiTemplate.java:87)
at org.springframework.jndi.JndiTemplate.lookup(JndiTemplate.java:152)
at org.springframework.jndi.JndiTemplate.lookup(JndiTemplate.java:179)
at org.springframework.jndi.JndiLocatorSupport.lookup(JndiLocatorSupport.java:95)
at org.springframework.jndi.JndiObjectLocator.lookup(JndiObjectLocator.java:106)
at org.springframework.jndi.JndiObjectTargetSource.getTarget(JndiObjectTargetSource.java:135)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:192)
at com.sun.proxy.$Proxy60.getConnection(Unknown Source)
at org.springframework.jdbc.datasource.DataSourceTransactionManager.doBegin(DataSourceTransactionManager.java:204)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373)
at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:130)
at org.springframework.session.jdbc.JdbcOperationsSessionRepository.cleanUpExpiredSessions(JdbcOperationsSessionRepository.java:475)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:65)
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54)
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.sql.SQLRecoverableException: IO Error: The Network Adapter could not establish the connection
at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:458)
at oracle.jdbc.driver.PhysicalConnection.<init>(PhysicalConnection.java:546)
at oracle.jdbc.driver.T4CConnection.<init>(T4CConnection.java:236)
at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:32)
at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:521)
at org.apache.tomcat.dbcp.dbcp.DriverConnectionFactory.createConnection(DriverConnectionFactory.java:38)
at org.apache.tomcat.dbcp.dbcp.PoolableConnectionFactory.makeObject(PoolableConnectionFactory.java:582)
at org.apache.tomcat.dbcp.dbcp.BasicDataSource.validateConnectionFactory(BasicDataSource.java:1558)
at org.apache.tomcat.dbcp.dbcp.BasicDataSource.createPoolableConnectionFactory(BasicDataSource.java:1547)
... 43 more
Caused by: oracle.net.ns.NetException: The Network Adapter could not establish the connection
at oracle.net.nt.ConnStrategy.execute(ConnStrategy.java:392)
at oracle.net.resolver.AddrResolution.resolveAndExecute(AddrResolution.java:434)
at oracle.net.ns.NSProtocol.establishConnection(NSProtocol.java:687)
at oracle.net.ns.NSProtocol.connect(NSProtocol.java:247)
at oracle.jdbc.driver.T4CConnection.connect(T4CConnection.java:1102)
at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:320)
... 51 more
Caused by: java.net.SocketTimeoutException: connect timed out
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:579)
at oracle.net.nt.TcpNTAdapter.connect(TcpNTAdapter.java:150)
at oracle.net.nt.ConnOption.connect(ConnOption.java:133)
at oracle.net.nt.ConnStrategy.execute(ConnStrategy.java:370)
... 56 more
The Code is running properly on one instance of tomcat but not on other three instances of tomcat.
We are using Oracle 12c database.
Please share your expert advice.
Any help will be appreciated.