We have a remote PC loaded with WebSphere 9.0 and we're trying to run an app that works in production, but we need to debug some things on the remote PC using Eclipse. We seem to have some kind of SSL problem with the app on the remote PC. We try to log in to the app, and get errors when it tries to get jquery - status 500. The jquery file it wants is definitely in the project.
Here is part of the logfile:
[1/18/23 12:02:30:787 EST] 000000fc webapp E com.ibm.ws.webcontainer.webapp.WebApp reportRecursiveError Error Page Exception:
[1/18/23 12:02:30:721 EST] 000000fc SystemErr R [/directory] - com.ait.security.servlets.ErrorS.displayError: (SSO artifact)
Request URI: GET /directory/lib/jquery-1.12.4.min.js
Remote addr: 0:0:0:0:0:0:0:1
Remote user: snagamalli
Referer: https://localhost:9443/directory/loginForm
Status code: 500
Status code: 500
Servlet name: Static File wrapper
Servlet error message: javax.servlet.ServletException: com.ibm.wsspi.webcontainer.ClosedConnectionException: OutputStream encountered error during write
Parameters:
_=1673975347000
javax.servlet.ServletException: com.ibm.wsspi.webcontainer.ClosedConnectionException: OutputStream encountered error during write
at com.ibm.ws.webcontainer.servlet.FileServletWrapper.service(FileServletWrapper.java:751)
at com.ibm.ws.webcontainer.servlet.FileServletWrapper.handleRequest(FileServletWrapper.java:543)
at com.ibm.ws.webcontainer.servlet.StaticFileServletWrapperImpl.handleRequest(StaticFileServletWrapperImpl.java:176)
at com.ibm.ws.webcontainer.extension.DefaultExtensionProcessor.handleRequest(DefaultExtensionProcessor.java:780)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:143)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:96)
at com.ait.security.filters.SecurityCheckFilter.doFilter(SecurityCheckFilter.java:99)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:197)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.ait.security.filters.PerformanceFilter.doFilter(PerformanceFilter.java:78)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:197)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.ait.web.html.ImproveCachingFilter.doFilter(ImproveCachingFilter.java:87)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:197)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:979)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1119)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:4219)
at com.ibm.ws.webcontainer.webapp.WebAppImpl.handleRequest(WebAppImpl.java:2210)
at com.ibm.ws.webcontainer.webapp.WebGroup.handleRequest(WebGroup.java:304)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:1030)
at com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1817)
at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:382)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:465)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewRequest(HttpInboundLink.java:532)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.processRequest(HttpInboundLink.java:318)
at com.ibm.ws.http.channel.inbound.impl.HttpICLReadCallback.complete(HttpICLReadCallback.java:88)
at com.ibm.ws.ssl.channel.impl.SSLReadServiceContext$SSLReadCompletedCallback.complete(SSLReadServiceContext.java:1833)
at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:175)
at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
at com.ibm.io.async.AsyncChannelFuture.fireCompletionActions(AsyncChannelFuture.java:161)
at com.ibm.io.async.AsyncFuture.completed(AsyncFuture.java:138)
at com.ibm.io.async.ResultHandler.complete(ResultHandler.java:204)
at com.ibm.io.async.ResultHandler.runEventProcessingLoop(ResultHandler.java:775)
at com.ibm.io.async.ResultHandler$2.run(ResultHandler.java:905)
at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1909)
Caused by: com.ibm.wsspi.webcontainer.ClosedConnectionException: OutputStream encountered error during write
at com.ibm.ws.webcontainer.channel.WCCByteBufferOutputStream.write(WCCByteBufferOutputStream.java:192)
at com.ibm.ws.webcontainer.srt.SRTOutputStream.write(SRTOutputStream.java:106)
at com.ibm.wsspi.webcontainer.util.BufferedServletOutputStream.writeOut(BufferedServletOutputStream.java:837)
at com.ibm.wsspi.webcontainer.util.BufferedServletOutputStream.writeThis(BufferedServletOutputStream.java:529)
at com.ibm.wsspi.webcontainer.util.BufferedServletOutputStream.write(BufferedServletOutputStream.java:438)
at com.ibm.ws.webcontainer.servlet.FileServletWrapper.writeByBytes(FileServletWrapper.java:1076)
at com.ibm.ws.webcontainer.servlet.FileServletWrapper._writeResponseToClient(FileServletWrapper.java:1035)
at com.ibm.ws.webcontainer.servlet.FileServletWrapper.writeResponseToClient(FileServletWrapper.java:944)
at com.ibm.ws.webcontainer.servlet.FileServletWrapper.service(FileServletWrapper.java:723)
... 35 more
Caused by: javax.net.ssl.SSLException: Fail to wrap application data
at com.ibm.jsse2.g.a(g.java:26)
at com.ibm.jsse2.bb.a(bb.java:102)
at com.ibm.jsse2.bb.a(bb.java:22)
at com.ibm.jsse2.bg.a(bg.java:308)
at com.ibm.jsse2.bg.wrap(bg.java:289)
at javax.net.ssl.SSLEngine.wrap(SSLEngine.java:37)
at com.ibm.ws.ssl.channel.impl.SSLWriteServiceContext.encryptMessage(SSLWriteServiceContext.java:619)
at com.ibm.ws.ssl.channel.impl.SSLWriteServiceContext.encryptAndWriteAsync(SSLWriteServiceContext.java:400)
at com.ibm.ws.ssl.channel.impl.SSLWriteServiceContext.write(SSLWriteServiceContext.java:331)
at com.ibm.ws.ssl.channel.impl.SSLWriteServiceContext.write(SSLWriteServiceContext.java:177)
at com.ibm.ws.http.channel.impl.HttpServiceContextImpl.asynchWrite(HttpServiceContextImpl.java:2523)
at com.ibm.ws.http.channel.impl.HttpServiceContextImpl.sendOutgoing(HttpServiceContextImpl.java:2307)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundServiceContextImpl.sendResponseBody(HttpInboundServiceContextImpl.java:902)
at com.ibm.ws.webcontainer.channel.WCChannelLink.writeBufferAsynch(WCChannelLink.java:744)
at com.ibm.ws.webcontainer.channel.WCChannelLink.writeBufferResponse(WCChannelLink.java:718)
at com.ibm.ws.webcontainer.channel.WCChannelLink.writeBuffer(WCChannelLink.java:655)
at com.ibm.ws.webcontainer.channel.WCCByteBufferOutputStream.flushWriteBuffer(WCCByteBufferOutputStream.java:723)
at com.ibm.ws.webcontainer.channel.WCCByteBufferOutputStream.checkWriteArray(WCCByteBufferOutputStream.java:631)
at com.ibm.ws.webcontainer.channel.WCCByteBufferOutputStream.write(WCCByteBufferOutputStream.java:197)
... 43 more
Caused by: java.security.ProviderException: Could not determine buffer size
at javax.crypto.CipherSpi.a(Unknown Source)
at javax.crypto.CipherSpi.engineDoFinal(Unknown Source)
at javax.crypto.Cipher.doFinal(Unknown Source)
at com.ibm.jsse2.n$o$a.a(n$o$a.java:61)
at com.ibm.jsse2.d.e(d.java:14)
at com.ibm.jsse2.d.a(d.java:123)
at com.ibm.jsse2.e.a(e.java:162)
at com.ibm.jsse2.e.a(e.java:130)
at com.ibm.jsse2.bg.c(bg.java:126)
at com.ibm.jsse2.bg.b(bg.java:274)
at com.ibm.jsse2.bg.a(bg.java:239)
... 58 more
Caused by: java.lang.IllegalStateException: Cipher has not been initialized
at com.ibm.crypto.plus.provider.ChaCha20Poly1305Cipher.a(ChaCha20Poly1305Cipher.java:144)
at com.ibm.crypto.plus.provider.ChaCha20Poly1305Cipher.engineDoFinal(ChaCha20Poly1305Cipher.java:187)
... 69 more
Apps that don't use SSL seem to work. We checked the default certificates in the WebSphere administrative console, but they seem to be ok - not expired.
I've installed WebSphere on developer's PCs many times, so don't think I missed anything, but that's certainly possible. Googling the various error messages doesn't seem to give us anything helpful, but the error messages seem explicit enough that they should lead us to the problem:
OutputStream encountered error during write...
Fail to wrap application data...
Could not determine buffer size...
Cipher has not been initialized...
Does anyone have any ideas about what may be wrong?
Related
I have WebSphere application server liberty - 17.0.0.2 installed. When i try to hit website URL after turning up crsServer, The front page displays "An error has occurred. Please contact your system administrator" message.
Initially everything was working fine but suddenly the storeFront stopped showing and there is below error printed in console.
SRVE0315E: An exception occurred: java.lang.Throwable: javax.servlet.ServletException: java.lang.NullPointerException
Here are the logs
[1/16/23 19:50:48:986 IST] 00000020 com.ibm.commerce.foundation.logging.ExtendedInfo I CWXFR9010I: Extended information : [URL=http://appsrv-05.kotsovolos.gr:8080/webapp/remote/preview/] [parameters=null] [userId=-1002]
[1/16/23 19:50:49:018 IST] 00000020 com.ibm.ws.logging.internal.impl.IncidentImpl I FFDC1015I: An FFDC Incident has been created: "javax.servlet.ServletException: java.lang.NullPointerException com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter 144" at ffdc_23.01.16_19.50.48.0.log
[1/16/23 19:50:49:049 IST] 00000020 com.ibm.ws.logging.internal.impl.IncidentImpl I FFDC1015I: An FFDC Incident has been created: "javax.servlet.ServletException: java.lang.NullPointerException com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter 82" at ffdc_23.01.16_19.50.49.0.log
[1/16/23 19:50:49:064 IST] 00000020 com.ibm.ws.logging.internal.impl.IncidentImpl I FFDC1015I: An FFDC Incident has been created: "javax.servlet.ServletException: java.lang.NullPointerException com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters 1064" at ffdc_23.01.16_19.50.49.1.log
[1/16/23 19:50:49:064 IST] 00000020 com.ibm.ws.webcontainer.webapp E SRVE0315E: An exception occurred: java.lang.Throwable: javax.servlet.ServletException: java.lang.NullPointerException
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:5027)
at com.ibm.ws.webcontainer31.osgi.webapp.WebApp31.handleRequest(WebApp31.java:528)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.handleRequest(DynamicVirtualHost.java:315)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:1025)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.run(DynamicVirtualHost.java:280)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink$TaskWrapper.run(HttpDispatcherLink.java:967)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.wrapHandlerAndExecute(HttpDispatcherLink.java:359)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.ready(HttpDispatcherLink.java:318)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:471)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleNewRequest(HttpInboundLink.java:405)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.processRequest(HttpInboundLink.java:285)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.ready(HttpInboundLink.java:256)
at com.ibm.ws.tcpchannel.internal.NewConnectionInitialReadCallback.sendToDiscriminators(NewConnectionInitialReadCallback.java:174)
at com.ibm.ws.tcpchannel.internal.NewConnectionInitialReadCallback.complete(NewConnectionInitialReadCallback.java:83)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.requestComplete(WorkQueueManager.java:504)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.attemptIO(WorkQueueManager.java:574)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.workerRun(WorkQueueManager.java:929)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager$Worker.run(WorkQueueManager.java:1018)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1153)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.lang.Thread.run(Thread.java:785)
Caused by: javax.servlet.ServletException: java.lang.NullPointerException
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter.doFilter(StoreRuntimeServletFilter.java:529)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
at com.ibm.commerce.store.preview.servlet.PreviewFilter.doFilter(PreviewFilter.java:97)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
at com.ibm.commerce.cf.servlet.ExecutionContextFilter.doFilter(ExecutionContextFilter.java:311)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:1021)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1143)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:4983)
... 20 more
Caused by: java.lang.NullPointerException
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter.doFilterAction(StoreRuntimeServletFilter.java:636)
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter$1.run(StoreRuntimeServletFilter.java:410)
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter.doFilter(StoreRuntimeServletFilter.java:433)
... 31 more
I am trying to solve this error but i am not able to find root cause. even in the official documentations i am not able to find information on SRVE0315E. can anyone guide me what can be issue??
It means that an exception was thrown when trying to call a servlet. In this case the exception is a NullPointerException thrown from a Servlet Filter. The exception stack in your question indicates exactly where this occurred:
Caused by: java.lang.NullPointerException
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter.doFilterAction(StoreRuntimeServletFilter.java:636)
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter$1.run(StoreRuntimeServletFilter.java:410)
at com.ibm.commerce.store.servlet.StoreRuntimeServletFilter.doFilter(StoreRuntimeServletFilter.java:433)
The com.ibm.commerce.store.servlet.StoreRuntimeServletFilter class is not a Liberty class and the source is not in the question so I cannot comment on what caused it to throw a NullPointerException, but you would need to either look to the source of that class, or contact the owner of it for more information on why this occurred.
I am running a grails project that has an android client. I migrated my grails project from my local postgres db to an online database server. After I run the grails app the response is for the first few minutes. After that I start getting this error. When I rerun the app the error disappears for the first few minutes then recurs. Note that this problem was non existent when I was using the local database. Here is the full stacktrace.
2018-07-13 10:01:37.571 ERROR --- [nio-8080-exec-8] o.h.engine.jdbc.spi.SqlExceptionHelper : This connection has been closed.
2018-07-13 10:01:37.586 ERROR --- [nio-8080-exec-8] o.g.web.errors.GrailsExceptionResolver : PSQLException occurred when processing request: [POST] /api/ - parameters:
email: fkamau#gmail.com
action: login
password: 1234
This connection has been closed.. Stacktrace follows:
java.lang.reflect.InvocationTargetException: null
at org.grails.core.DefaultGrailsControllerClass$ReflectionInvoker.invoke(DefaultGrailsControllerClass.java:211)
at org.grails.core.DefaultGrailsControllerClass.invoke(DefaultGrailsControllerClass.java:188)
at org.grails.web.mapping.mvc.UrlMappingsInfoHandlerAdapter.handle(UrlMappingsInfoHandlerAdapter.groovy:90)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at org.springframework.boot.web.filter.ApplicationContextHeaderFilter.doFilterInternal(ApplicationContextHeaderFilter.java:55)
at org.grails.web.servlet.mvc.GrailsWebRequestFilter.doFilterInternal(GrailsWebRequestFilter.java:77)
at org.grails.web.filters.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:67)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.hibernate.exception.JDBCConnectionException: could not prepare statement
at org.hibernate.exception.internal.SQLStateConversionDelegate.convert(SQLStateConversionDelegate.java:115)
at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:42)
at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:111)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$StatementPreparationTemplate.prepareStatement(StatementPreparerImpl.java:182)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl.prepareQueryStatement(StatementPreparerImpl.java:148)
at org.hibernate.loader.Loader.prepareQueryStatement(Loader.java:1934)
at org.hibernate.loader.Loader.executeQueryStatement(Loader.java:1903)
at org.hibernate.loader.Loader.executeQueryStatement(Loader.java:1881)
at org.hibernate.loader.Loader.doQuery(Loader.java:925)
at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:342)
at org.hibernate.loader.Loader.doList(Loader.java:2622)
at org.hibernate.loader.Loader.doList(Loader.java:2605)
at org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2434)
at org.hibernate.loader.Loader.list(Loader.java:2429)
at org.hibernate.loader.criteria.CriteriaLoader.list(CriteriaLoader.java:109)
at org.hibernate.internal.SessionImpl.list(SessionImpl.java:1787)
at org.hibernate.internal.CriteriaImpl.list(CriteriaImpl.java:363)
at org.grails.orm.hibernate.query.AbstractHibernateQuery.singleResultViaListCall(AbstractHibernateQuery.java:801)
at org.grails.orm.hibernate.query.AbstractHibernateQuery.singleResult(AbstractHibernateQuery.java:791)
at org.grails.datastore.gorm.finders.AbstractFindByFinder.invokeQuery(AbstractFindByFinder.java:35)
at org.grails.datastore.gorm.finders.AbstractFindByFinder$1.doInSession(AbstractFindByFinder.java:29)
at org.grails.datastore.mapping.core.DatastoreUtils.execute(DatastoreUtils.java:319)
at org.grails.datastore.gorm.finders.AbstractFinder.execute(AbstractFinder.java:42)
at org.grails.datastore.gorm.finders.AbstractFindByFinder.doInvokeInternal(AbstractFindByFinder.java:27)
at org.grails.datastore.gorm.finders.DynamicFinder.invoke(DynamicFinder.java:252)
at org.grails.datastore.gorm.finders.DynamicFinder.invoke(DynamicFinder.java:390)
at org.grails.datastore.gorm.GormStaticApi$_methodMissing_closure2.doCall(GormStaticApi.groovy:181)
at eafyaportal.PatientService.login(PatientService.groovy:55)
at eafyaportal.ApiController.login(ApiController.groovy:43)
... 14 common frames omitted
Caused by: org.postgresql.util.PSQLException: This connection has been closed.
at org.postgresql.jdbc2.AbstractJdbc2Connection.checkClosed(AbstractJdbc2Connection.java:714)
at org.postgresql.jdbc2.AbstractJdbc2Connection.getAutoCommit(AbstractJdbc2Connection.java:680)
at org.springframework.jdbc.datasource.LazyConnectionDataSourceProxy$LazyConnectionInvocationHandler.getTargetConnection(LazyConnectionDataSourceProxy.java:422)
at org.springframework.jdbc.datasource.LazyConnectionDataSourceProxy$LazyConnectionInvocationHandler.invoke(LazyConnectionDataSourceProxy.java:376)
at org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy$TransactionAwareInvocationHandler.invoke(TransactionAwareDataSourceProxy.java:240)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$5.doPrepare(StatementPreparerImpl.java:146)
at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$StatementPreparationTemplate.prepareStatement(StatementPreparerImpl.java:172)
... 39 common frames omitted
Your problem sounds related to the connection pool abandoning connections. Is there a specific connection pooling implementation that you are using? One of the most commonly used is Hikari. It has options to testOnBorrow which should check if a connection is alive before using it.
How do I configure HikariCP in my Spring Boot app in my application.properties files?
I have recently run into a problem while trying to access my database through a java web application. I keep getting a CannotGetJdbcConnectionException along with a StaleConnectionException. I am using Websphere v8.5 as my local machine server. I have tried testing the Data Source Connection in the Administrative console and it always comes back successful. But whenever I try to access my application I get the above Exceptions. Does anyone have an idea of why this keeps happening and how I could possibly fix it? Below is the error message I keep getting.
I am using DB2 for my database
[10/1/15 7:47:23:550 CDT] 00000073 FfdcProvider W com.ibm.ws.ffdc.impl.FfdcProvider logIncident FFDC1003I: FFDC Incident emitted on C:\Dev2\Utilities\IBM\AppServers\profiles\AppSrv03\logs\ffdc\server1_ef037f_15.10.01_07.47.23.5493440764669613130910.txt com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest() 309
[10/1/15 7:47:23:553 CDT] 00000073 webapp E com.ibm.ws.webcontainer.webapp.WebApp logServletError SRVE0293E: [Servlet Error]-[mvc-dispatcher]: org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is com.ibm.websphere.ce.cm.StaleConnectionException: null DSRA0010E: SQL State = 56038, Error Code = -4,700
at org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:80)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:628)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:693)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:720)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:770)
at org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate.query(NamedParameterJdbcTemplate.java:190)
at org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate.query(NamedParameterJdbcTemplate.java:202)
at com.shelter.trac.dao.AgencyDao.getStatesNDistricts(AgencyDao.java:32)
at com.shelter.trac.serviceManager.AgencyManager.getStatesNDistricts(AgencyManager.java:27)
at com.shelter.trac.controller.ApiController.getStatesNDistricts(ApiController.java:40)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
at java.lang.reflect.Method.invoke(Method.java:613)
at org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:214)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:132)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:104)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:748)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:689)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:83)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:945)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:876)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:931)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:822)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:575)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:807)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:668)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1214)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:774)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:456)
at com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:178)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1027)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:3703)
at com.ibm.ws.webcontainer.webapp.WebGroup.handleRequest(WebGroup.java:304)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:962)
at com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1662)
at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:195)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:458)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewRequest(HttpInboundLink.java:522)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.processRequest(HttpInboundLink.java:311)
at com.ibm.ws.http.channel.inbound.impl.HttpICLReadCallback.complete(HttpICLReadCallback.java:87)
at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:165)
at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
at com.ibm.io.async.AsyncChannelFuture.fireCompletionActions(AsyncChannelFuture.java:161)
at com.ibm.io.async.AsyncFuture.completed(AsyncFuture.java:138)
at com.ibm.io.async.ResultHandler.complete(ResultHandler.java:204)
at com.ibm.io.async.ResultHandler.runEventProcessingLoop(ResultHandler.java:775)
at com.ibm.io.async.ResultHandler$2.run(ResultHandler.java:905)
at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1783)
Caused by: com.ibm.websphere.ce.cm.StaleConnectionException: null DSRA0010E: SQL State = 56038, Error Code = -4,700
at com.ibm.ws.rsadapter.spi.ServerFunction.handleStaleStatement(ServerFunction.java:693)
at com.ibm.ws.rsadapter.AdapterUtil.mapException(AdapterUtil.java:2277)
at com.ibm.ws.rsadapter.AdapterUtil.translateSQLException(AdapterUtil.java:1582)
at com.ibm.ws.rsadapter.spi.WSRdbDataSource.getConnection(WSRdbDataSource.java:2243)
at com.ibm.ws.rsadapter.spi.WSManagedConnectionFactoryImpl.getConnection(WSManagedConnectionFactoryImpl.java:1761)
at com.ibm.ws.rsadapter.spi.WSManagedConnectionFactoryImpl.createManagedConnection(WSManagedConnectionFactoryImpl.java:1506)
at com.ibm.ws.rsadapter.spi.WSManagedConnectionFactoryImpl.createManagedConnection(WSManagedConnectionFactoryImpl.java:1096)
at com.ibm.ejs.j2c.FreePool.createManagedConnectionWithMCWrapper(FreePool.java:2086)
at com.ibm.ejs.j2c.FreePool.createOrWaitForConnection(FreePool.java:1766)
at com.ibm.ejs.j2c.PoolManager.reserve(PoolManager.java:3264)
at com.ibm.ejs.j2c.PoolManager.reserve(PoolManager.java:2567)
at com.ibm.ejs.j2c.ConnectionManager.allocateMCWrapper(ConnectionManager.java:1544)
at com.ibm.ejs.j2c.ConnectionManager.allocateConnection(ConnectionManager.java:1027)
at com.ibm.ws.rsadapter.jdbc.WSJdbcDataSource.getConnection(WSJdbcDataSource.java:642)
at com.ibm.ws.rsadapter.jdbc.WSJdbcDataSource.getConnection(WSJdbcDataSource.java:609)
at org.springframework.jdbc.datasource.DataSourceUtils.doGetConnection(DataSourceUtils.java:111)
at org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:77)
... 48 more
I did some more research on the subject especially the StaleConnectionException, and most of the article talked about the JDBC exceptions timing out or going stale. So I went into Websphere Administration Console and started looking around for properties about connections. In the Resources -> JDBC -> Data source -> [your Datasource] there is a page about "Websphere Application Server Data Source Propeties" and in it is a section called "Connection Validation Properties". I decided to give it a shot and checked the Validate New Connections checkbox and left the Number of Retries and Retry interval to their default settings and tested the app and it worked.
The scenario is,
We created a Webservice (say WS1) using WebExperience factory V8.0.0.2 and the websphere application server is V8.0.0.10. The WS1 has an operation which calls another webservice (say WS2) for some authentication. All are soap services. When testing this scenario locally we were able to invoke the underlying webservice WS2 and was able to get the response.
I deployed it in Webphere Portal, wp_profile, but when I called the service WS1 it is showing some SOAP error. The error is happening when WS1 invokes WS2. We are using Portal 8.0 and the Application Server version is 8.0.0.10.
I changed the class loading to parent last in the application module. But it is still showing the error.
Stack trace is given
Category: bowstreet.system.server.logging.event.criterion.abnormal
Priority: ERROR
Msg: Component: SERVICE
Caught an exception processing a message style web service call.
The associated exception is: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
at java.lang.reflect.Method.invoke(Method.java:611)
at org.apache.axis.providers.java.MsgProvider.processMessage(MsgProvider.java:141)
at com.bowstreet.axis.providers.WebAppMSGProvider.processMessage(WebAppMSGProvider.java:46)
at org.apache.axis.providers.java.JavaProvider.invoke(JavaProvider.java:323)
at org.apache.axis.strategies.InvocationStrategy.visit(InvocationStrategy.java:32)
at org.apache.axis.SimpleChain.doVisiting(SimpleChain.java:118)
at org.apache.axis.SimpleChain.invoke(SimpleChain.java:83)
at org.apache.axis.handlers.soap.SOAPService.invoke(SOAPService.java:454)
at org.apache.axis.server.AxisServer.invoke(AxisServer.java:281)
at org.apache.axis.transport.http.AxisServlet.doPost(AxisServlet.java:699)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:595)
at org.apache.axis.transport.http.AxisServletBase.service(AxisServletBase.java:327)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:668)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1214)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:774)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:456)
at com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:178)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:125)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:77)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:926)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1023)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:3703)
at com.ibm.ws.webcontainer.webapp.WebGroup.handleRequest(WebGroup.java:304)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:962)
at com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1662)
at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:195)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:452)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewRequest(HttpInboundLink.java:511)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.processRequest(HttpInboundLink.java:305)
at com.ibm.ws.http.channel.inbound.impl.HttpICLReadCallback.complete(HttpICLReadCallback.java:83)
at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:165)
at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
at com.ibm.io.async.AsyncChannelFuture.fireCompletionActions(AsyncChannelFuture.java:161)
at com.ibm.io.async.AsyncFuture.completed(AsyncFuture.java:138)
at com.ibm.io.async.ResultHandler.complete(ResultHandler.java:204)
at com.ibm.io.async.ResultHandler.runEventProcessingLoop(ResultHandler.java:775)
at com.ibm.io.async.ResultHandler$2.run(ResultHandler.java:905)
at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1659)
Caused by: java.rmi.RemoteException: Error in method BatchInvoke_al. Error in method getDetail_ws.invoke. Error in method getDetail_ws.invokeInternal. javax/xml/soap/MessageFactory.newInstance(Ljava/lang/String;)Ljavax/xml/soap/MessageFactory;
at com.bowstreet.axis.WebAppRPCInterface.aMethod(WebAppRPCInterface.java:68)
... 41 more
Can anyone help me on this issue..
The WARs you're deploying to the two servers (working and non-working) are different one is dev mode WAR built by deploying from designer (which is a dual mode WAR that supports the WEF standalone servlet mode plus portlets) and the other an exported portlet WAR. If so, then the two WARs would have different sets of JARs (less jars in the portlet WAR which defers to portal for some of those APIs).
Please cross checked the module class loader in the application module.
saaj-api.jar will be missing in dev mode war bulid.
Remove the saaj-api.jar from the lib and export the war and deploy.
Can you provide the libraries using in the Applications?
Please check whether any dependant jar files is loading twice after starting the application.
You can view the loaded class files in the application server console.
We have a JAX-WS servce on one node of our WebSphere 8.5.5 box, and it needs to talk to a JAX-WS service on another node on the same box. It's been doing this in WebSpere 7 for months.
Yet suddenly, it won't. I wish I could say that ONLY the server version changed, but no, lots of code changed. But that code is now working on a WebSphere 7 box. So it seems like it's a version thing.
I can call the service that is being called by the other service with a unit test and it works fine.
The called service does not seem to report an exception. The calling service complains:
[8/29/14 14:59:49:222 EDT] 0000007d AxisEngine E org.apache.axis2.engine.AxisEngine receive The endpoint reference (EPR) for the Operation not found is {correct URL redacted) and the WSA Action = . If this EPR was previously reachable, please contact the server administrator.
org.apache.axis2.AxisFault: The endpoint reference (EPR) for the Operation not found is (correct URL redacted) and the WSA Action = . If this EPR was previously reachable, please contact the server administrator.
at org.apache.axis2.engine.DispatchPhase.checkPostConditions(DispatchPhase.java:102)
at org.apache.axis2.engine.Phase.invoke(Phase.java:329)
at org.apache.axis2.engine.AxisEngine.invoke(AxisEngine.java:360)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:195)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:172)
at com.ibm.ws.websvcs.transport.http.WASAxis2Servlet.doPost(WASAxis2Servlet.java:1583)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:595)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:668)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1230)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:779)
at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:478)
at com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:178)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:136)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:97)
at org.apache.logging.log4j.core.web.Log4jServletFilter.doFilter(Log4jServletFilter.java:66)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:960)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1064)
at com.ibm.ws.webcontainer.servlet.CacheServletWrapper.handleRequest(CacheServletWrapper.java:87)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:909)
at com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1662)
at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:200)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:459)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewRequest(HttpInboundLink.java:526)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.processRequest(HttpInboundLink.java:312)
at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.ready(HttpInboundLink.java:283)
at com.ibm.ws.tcp.channel.impl.NewConnectionInitialReadCallback.sendToDiscriminators(NewConnectionInitialReadCallback.java:214)
at com.ibm.ws.tcp.channel.impl.NewConnectionInitialReadCallback.complete(NewConnectionInitialReadCallback.java:113)
at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:175)
at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
at com.ibm.io.async.AsyncChannelFuture.fireCompletionActions(AsyncChannelFuture.java:161)
at com.ibm.io.async.AsyncFuture.completed(AsyncFuture.java:138)
at com.ibm.io.async.ResultHandler.complete(ResultHandler.java:204)
at com.ibm.io.async.ResultHandler.runEventProcessingLoop(ResultHandler.java:775)
at com.ibm.io.async.ResultHandler$2.run(ResultHandler.java:905)
at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1862)
So here's what happened. Actually about half a dozen things went wrong at the same time which lead to people looking for solutions in the wrong places. But this particular error seemed to have come from a set of .class files that were not checked in in the .apt_generated folder after a recent change to the interface of the web service being called.
Once checked in, the deployment workspace got them, and all worked.