I am trying to setup SOLR for Sitecore with pretty big content tree (~300k items, ~25 GB Database size).
When I try to rebuild biggest indexes (like sitecore_web_index and sitecore_master_index) my SOLR instance goes down with error:
<lst name="responseHeader"><int name="status">500</int><int name="QTime">1</int></lst><lst name="initFailures"/><lst name="error"><lst name="metadata"><str name="error-class">org.apache.solr.common.SolrException</str><str name="root-error-class">java.nio.file.NoSuchFileException</str></lst><str name="msg">Error handling 'status' action </str><str name="trace">org.apache.solr.common.SolrException: Error handling 'status' action
at org.apache.solr.handler.admin.CoreAdminOperation$4.call(CoreAdminOperation.java:192)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:354)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:153)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:155)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:676)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:439)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:225)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:183)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.file.NoSuchFileException: /var/solr/data/sitecore_master_index/data/index/segments_90
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:55)
at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
at sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
at java.nio.file.Files.readAttributes(Files.java:1737)
at java.nio.file.Files.size(Files.java:2332)
at org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:210)
at org.apache.lucene.store.NRTCachingDirectory.fileLength(NRTCachingDirectory.java:127)
at org.apache.solr.handler.admin.LukeRequestHandler.getIndexInfo(LukeRequestHandler.java:592)
at org.apache.solr.handler.admin.CoreAdminOperation.getCoreStatus(CoreAdminOperation.java:886)
at org.apache.solr.handler.admin.CoreAdminOperation$4.call(CoreAdminOperation.java:188)
... 27 more
VM for SOLR have 8 Gb RAM, enough HDD space, 4 vCPU's assigned.
Java options is -Xms1024m -Xmx7168m
I shall note, that this failure is permament, and SOLR will not come back alive until it will be restarted manually.
Related
Locally the api from the plan works, but when it deploys it throws this exception:
com.google.apphosting.runtime.jetty9.JettyLogger warn: /_ah/api/esb/v1/activity-job (JettyLogger.java:45)
java.lang.NullPointerException
at com.google.api.server.spi.EndpointsServlet.service(EndpointsServlet.java:72)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:848)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1780)
at com.google.api.control.ControlFilter.doFilter(ControlFilter.java:331)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1767)
at com.google.api.control.ConfigFilter.doFilter(ConfigFilter.java:120)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1767)
at com.google.apphosting.utils.servlet.JdbcMySqlConnectionCleanupFilter.doFilter(JdbcMySqlConnectionCleanupFilter.java:60)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1767)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:583)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:524)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at com.google.apphosting.runtime.jetty9.ParseBlobUploadHandler.handle(ParseBlobUploadHandler.java:124)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1182)
at com.google.apphosting.runtime.jetty9.AppEngineWebAppContext.doHandle(AppEngineWebAppContext.java:187)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:513)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at com.google.apphosting.runtime.jetty9.AppVersionHandlerMap.handle(AppVersionHandlerMap.java:293)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:539)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333)
at com.google.apphosting.runtime.jetty9.RpcConnection.handle(RpcConnection.java:216)
at com.google.apphosting.runtime.jetty9.RpcConnector.serviceRequest(RpcConnector.java:81)
at com.google.apphosting.runtime.jetty9.JettyServletEngineAdapter.serviceRequest(JettyServletEngineAdapter.java:134)
at com.google.apphosting.runtime.RequestRunner.dispatchServletRequest(RequestRunner.java:264)
at com.google.apphosting.runtime.RequestRunner.dispatchRequest(RequestRunner.java:229)
at com.google.apphosting.runtime.RequestRunner.run(RequestRunner.java:194)
at com.google.apphosting.runtime.ThreadGroupPool$PoolEntry.run(ThreadGroupPool.java:273)
at java.lang.Thread.run(Thread.java:748)
please help in giving the resolution this error is comming when we are deploying when debuging locally it works fine
I have a java.lang.ExceptionInInitializerError when trying to execute BufferedImage originalImage = ImageIO.read(in);. This happens only on a production VM, not on my dev machine.
I created a small class with the above line on the production VM, compiled and executed it successfully. Then I tried adding java options "-Djava.awt.headless=true -Dawt.toolkit=sun.awt.HToolkit" as suggested here. Didn't help. Then I tried adding static {ImageIO.scanForPlugins();} as suggested here. Also didn't help. I ran jHades to try to see if there were any overlapping classes in jars. There are none.
Now I'm out of ideas. The dev machine is running java 1.8.0_76-release-b198 on Ubuntu 14.04 and the production VM runs java 1.8.0_60-b27 on CentOS. Any pointers?
Update: Full stack trace:
java.lang.ExceptionInInitializerError
at com.comp.core.model.artifactThumbnails.ArtifactThumbnailRepository.generateThumbnail(ArtifactThumbnailRepository.java:63)
at com.comp.core.model.artifactThumbnails.BigTableArtifactThumbnailRepository.createThumbnail(BigTableArtifactThumbnailRepository.java:71)
at com.comp.web.routes.vertex.VertexThumbnail.handle(VertexThumbnail.java:90)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.web.privilegeFilters.PrivilegeFilter.handle(PrivilegeFilter.java:35)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.miniweb.handlers.CSRFHandler.handle(CSRFHandler.java:66)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.web.AuthenticationHandler.handle(AuthenticationHandler.java:17)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.miniweb.Router.dispatch(Router.java:93)
at com.comp.miniweb.Router.routeWithExceptionHandling(Router.java:87)
at com.comp.miniweb.Router.route(Router.java:44)
at com.comp.miniweb.App.handle(App.java:128)
at com.comp.web.WebApp.handle(WebApp.java:71)
at com.comp.web.Router.service(Router.java:186)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:800)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)
at com.comp.web.RequestDebugFilter.doFilter(RequestDebugFilter.java:33)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.websocket.server.WebSocketUpgradeFilter.doFilter(WebSocketUpgradeFilter.java:224)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:497)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:245)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
Caused by:
java.lang.SecurityException: sealing violation: package com.sun.media.imageioimpl.plugins.jpeg is sealed
at java.net.URLClassLoader.getAndVerifyPackage(URLClassLoader.java:399)
at java.net.URLClassLoader.definePackageInternal(URLClassLoader.java:419)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:451)
at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at org.eclipse.jetty.webapp.WebAppClassLoader.findClass(WebAppClassLoader.java:510)
at org.eclipse.jetty.webapp.WebAppClassLoader.loadClass(WebAppClassLoader.java:441)
at org.eclipse.jetty.webapp.WebAppClassLoader.loadClass(WebAppClassLoader.java:403)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:370)
at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
at javax.imageio.spi.IIORegistry.registerApplicationClasspathSpis(IIORegistry.java:210)
at javax.imageio.spi.IIORegistry.<init>(IIORegistry.java:138)
at javax.imageio.spi.IIORegistry.getDefaultInstance(IIORegistry.java:159)
at javax.imageio.ImageIO.<clinit>(ImageIO.java:66)
at com.comp.core.model.artifactThumbnails.ArtifactThumbnailRepository.generateThumbnail(ArtifactThumbnailRepository.java:63)
at com.comp.core.model.artifactThumbnails.BigTableArtifactThumbnailRepository.createThumbnail(BigTableArtifactThumbnailRepository.java:71)
at com.comp.web.routes.vertex.VertexThumbnail.handle(VertexThumbnail.java:90)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.web.privilegeFilters.PrivilegeFilter.handle(PrivilegeFilter.java:35)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.miniweb.handlers.CSRFHandler.handle(CSRFHandler.java:66)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.web.AuthenticationHandler.handle(AuthenticationHandler.java:17)
at com.comp.miniweb.HandlerChain.next(HandlerChain.java:18)
at com.comp.miniweb.Router.dispatch(Router.java:93)
at com.comp.miniweb.Router.routeWithExceptionHandling(Router.java:87)
at com.comp.miniweb.Router.route(Router.java:44)
at com.comp.miniweb.App.handle(App.java:128)
at com.comp.web.WebApp.handle(WebApp.java:71)
at com.comp.web.Router.service(Router.java:186)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:800)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)
at com.comp.web.RequestDebugFilter.doFilter(RequestDebugFilter.java:33)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.websocket.server.WebSocketUpgradeFilter.doFilter(WebSocketUpgradeFilter.java:224)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:497)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:245)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
My jetty server version is "9.3.10.v20160621".Below is stack trace--
org.eclipse.jetty.io.RuntimeIOException: org.eclipse.jetty.io.EofException
at org.eclipse.jetty.server.ResponseWriter.isOpen(ResponseWriter.java:133)
at org.eclipse.jetty.server.ResponseWriter.println(ResponseWriter.java:414)
at com.abc.servlet.BedListAdminServlet.handleChooseFromBedsOnNetwork(BedListAdminServlet.java:929)
at com.abc.servlet.BedListAdminServlet.handleEvent(BedListAdminServlet.java:322)
at com.abc.servlet.BaseServletEventHandler.doPost(BaseServletEventHandler.java:56)
at com.abc.servlet.BedListAdminServlet.doFilteredGet(BedListAdminServlet.java:293)
at com.abc.servlet.BaseServletEventHandler.doGet(BaseServletEventHandler.java:30)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:583)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.eclipse.jetty.io.EofException
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:197)
at org.eclipse.jetty.io.WriteFlusher.flush(WriteFlusher.java:420)
at org.eclipse.jetty.io.WriteFlusher.write(WriteFlusher.java:313)
at org.eclipse.jetty.io.AbstractEndPoint.write(AbstractEndPoint.java:140)
at org.eclipse.jetty.server.HttpConnection$SendCallback.process(HttpConnection.java:741)
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:241)
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:224)
at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:521)
at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:697)
at org.eclipse.jetty.server.HttpChannel.write(HttpChannel.java:746)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:179)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:163)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:441)
at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:167)
at org.eclipse.jetty.server.Utf8HttpWriter.write(Utf8HttpWriter.java:183)
at org.eclipse.jetty.server.HttpWriter.write(HttpWriter.java:71)
at org.eclipse.jetty.server.HttpWriter.write(HttpWriter.java:65)
at org.eclipse.jetty.server.ResponseWriter.println(ResponseWriter.java:415)
at com.abc.servlet.BedListAdminServlet.handleChooseFromBedsOnNetwork(BedListAdminServlet.java:927)
... 29 more
Caused by: java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.writev0(Native Method)
at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51)
at sun.nio.ch.IOUtil.write(IOUtil.java:148)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504)
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:177)
... 47 more
I'm facing this issue only CentOS 7, when I run the same on Windows laptop, I don't face any exceptions. Any help, will be appreciated..
You have to add
ContextHandler.setMaxFormContentSize(int maxSizeInBytes);
ContextHandler.setMaxFormKeys(int formKeys);
in your java class then you will not get that exception.
You can solve this problem in three ways one with vm arguments,Java code and through configuration xml files.
Below is the link from jetty
http://www.eclipse.org/jetty/documentation/9.3.x/configuring-form-size.html
Couldnt find whether there is any configuration for static content. Its not downloading the js file, I hope it because of larger in size when compare with others.
Jetty version : 9.2.10
OS : HP Nonstop
I can see the following errors in jetty debug
DEBUG org.eclipse.jetty.io.WriteFlusher - write exception
org.eclipse.jetty.io.EofException: null
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:192) ~[jetty-io-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.io.WriteFlusher.flush(WriteFlusher.java:408) ~[jetty-io-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.io.WriteFlusher.write(WriteFlusher.java:302) ~[jetty-io-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.io.AbstractEndPoint.write(AbstractEndPoint.java:129) ~[jetty-io-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpConnection$SendCallback.process(HttpConnection.java:676) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:246) ~[jetty-util-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:208) ~[jetty-util-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:471) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:763) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpChannel.write(HttpChannel.java:801) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpOutput.sendContent(HttpOutput.java:583) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpOutput.sendContent(HttpOutput.java:669) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpOutput.sendContent(HttpOutput.java:571) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.servlet.DefaultServlet.sendData(DefaultServlet.java:955) ~[jetty-servlet-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.servlet.DefaultServlet.doGet(DefaultServlet.java:533) ~[jetty-servlet-9.2.10.v20150310.jar:9.2.10.v20150310]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687) ~[servlet-api-3.1.jar:3.1.0]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) ~[servlet-api-3.1.jar:3.1.0]
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:808) ~[jetty-servlet-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) ~[jetty-servlet-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577) ~[jetty-security-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) ~[jetty-servlet-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.Server.handle(Server.java:497) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) ~[jetty-server-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) ~[jetty-io-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) ~[jetty-util-9.2.10.v20150310.jar:9.2.10.v20150310]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) ~[jetty-util-9.2.10.v20150310.jar:9.2.10.v20150310]
at java.lang.Thread.run(Thread.java:724) ~[na:1.7.0_25]
Caused by: java.io.IOException: Invalid function argument (errno:4022)
at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) ~[na:1.7.0_25]
at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) ~[na:1.7.0_25]
at sun.nio.ch.IOUtil.write(IOUtil.java:149) ~[na:1.7.0_25]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:512) ~[na:1.7.0_25]
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:172) ~[jetty-io-9.2.10.v20150310.jar:9.2.10.v20150310]
... 37 common frames omitted
You mean Jetty is uploading or sending the content, not downloading.
That much is evident from your stacktrace.
The main problem you are having is that the connection is terminated with an EOF when attempting to flush out that connection.
However, the more fundamental problem you are having java.io.IOException: Invalid function argument (errno:4022) isn't something that Jetty can fix or work around. This is an OS level error indicating either an inability to use the file descriptor to the network connection, or a file system error indicating an inability to use the file descriptor to access the file on disk.
Either way you should check your OS, the filesystem, the health of the WAR file, the filesystem temp directory, and also consider an upgrade of your Java JVM (1.7.0_25 expired back in Nov 15, 2013)
I'm running a high volume java web-app over a jetty 9.x server (4k-5k qps).
My servlet (MyServlet.java) is simple Java class the extends HttpServlet.
When getting the request object HttpServletRequest I invoke the getInputStream() method (as can be seen in the below stack) and then getting the below IOException.
I know that 'Broken pipe' means that the client closed the connection but when calculating the amount of time the request spent in the server I get ti 1ms and my client timeout is configured for much more.
Could this be the result of too much load? What could cause it? Is there some configuration in Jetty which I can play with that might affect this?
org.eclipse.jetty.io.EofException
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:192)
at org.eclipse.jetty.io.WriteFlusher.flush(WriteFlusher.java:408)
at org.eclipse.jetty.io.WriteFlusher.write(WriteFlusher.java:302)
at org.eclipse.jetty.io.AbstractEndPoint.write(AbstractEndPoint.java:129)
at org.eclipse.jetty.server.HttpConnection$SendCallback.process(HttpConnection.java:679)
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:246)
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:208)
at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:471)
at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:763)
at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:781)
at org.eclipse.jetty.server.HttpChannel.continue100(HttpChannel.java:240)
at org.eclipse.jetty.server.Request.getInputStream(Request.java:674)
at com.ta.btr.web.servlet.MyServlet.handlePostRequest(MyServlet.java:162)
at com.ta.btr.web.servlet.MyServlet.doPost(MyServlet.java:92)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:808)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:65)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:170)
... 35 more
It appears I had a problem in the load balancer which delayed the requests causing the client to close the connection (due to timeout).