text file upload (larger record numbers)
Hi,
quick note: I have been playing a bit with text file uploads for occurrence records, using a tabular export from the GBIF index on a 500,000+ record dataset. Using different subsets of the same recordset, I have not been able to get beyond the 310,000 records cutdown of the test set so far - this equals a file size of 99,974kB in the given example. The next larger set (320,000 records, 100MB) and all larger ones fail with
Yikes!
javax.servlet.ServletException: Expression resource is undefined on line 87, column 30 in WEB-INF/pages/manage/sources.ftl.
at net.sf.ehcache.constructs.web.filter.Filter.logThrowable(Filter.java:152)
at net.sf.ehcache.constructs.web.filter.Filter.doFilter(Filter.java:97)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application FilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh ain.java:188)
at org.appfuse.webapp.filter.LocaleFilter.doFilterInternal(LocaleFilter.java:74 )
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestF ilter.java:76)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application FilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh ain.java:188)
at org.springframework.orm.hibernate3.support.OpenSessionInViewFilter.doFilterI nternal(OpenSessionInViewFilter.java:198)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestF ilter.java:76)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application FilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh ain.java:188)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:359)
at org.springframework.security.intercept.web.FilterSecurityInterceptor.invoke( FilterSecurityInterceptor.java:109)
at org.springframework.security.intercept.web.FilterSecurityInterceptor.doFilte r(FilterSecurityInterceptor.java:83)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.ui.ExceptionTranslationFilter.doFilterHttp(Exce ptionTranslationFilter.java:101)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.providers.anonymous.AnonymousProcessingFilter.d oFilterHttp(AnonymousProcessingFilter.java:105)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.ui.rememberme.RememberMeProcessingFilter.doFilt erHttp(RememberMeProcessingFilter.java:116)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.wrapper.SecurityContextHolderAwareRequestFilter .doFilterHttp(SecurityContextHolderAwareRequestFilter.java:91)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.ui.basicauth.BasicProcessingFilter.doFilterHttp (BasicProcessingFilter.java:172)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.ui.AbstractProcessingFilter.doFilterHttp(Abstra ctProcessingFilter.java:268)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilte r.java:87)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.ui.SessionFixationProtectionFilter.doFilterHttp (SessionFixationProtectionFilter.java:61)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.context.HttpSessionContextIntegrationFilter.doF ilterHttp(HttpSessionContextIntegrationFilter.java:235)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity Filter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt er(FilterChainProxy.java:371)
at org.springframework.security.util.FilterChainProxy.doFilter(FilterChainProxy .java:174)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(Delegati ngFilterProxy.java:236)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilt erProxy.java:167)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application FilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh ain.java:188)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(Char acterEncodingFilter.java:96)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestF ilter.java:76)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application FilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh ain.java:188)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.ja va:210)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.ja va:174)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127 )
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117 )
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java :108)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:151)
at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:200)
at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:773)
at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:703)
at org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java :895)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.jav a:685)
at java.lang.Thread.run(Thread.java:595)
---------------------------------------------------------------
Andrea Hahn (ahahn@gbif.org)
Data Portal Manager
Global Biodiversity Information Facility Secretariat
Universitetsparken 15, DK-2100 Copenhagen, Denmark
Tel: +45-35321491 Fax: +45-35321480
URL: http://www.gbif.org
---------------------------------------------------------------
Andrea, the upload of files is limited to 100MB at the GBIF installation. We might want to increase it, but it might be better to allow upload of compressed files. what do you think?
markus
On Feb 27, 2009, at 4:57 PM, Andrea Hahn (GBIF) wrote:
Hi,
quick note: I have been playing a bit with text file uploads for occurrence records, using a tabular export from the GBIF index on a 500,000+ record dataset. Using different subsets of the same recordset, I have not been able to get beyond the 310,000 records cutdown of the test set so far - this equals a file size of 99,974kB in the given example. The next larger set (320,000 records, 100MB) and all larger ones fail with Yikes!
javax.servlet.ServletException: Expression resource is undefined on line 87, column 30 in WEB-INF/pages/manage/sources.ftl. at net.sf.ehcache.constructs.web.filter.Filter.logThrowable(Filter.java: 152) at net.sf.ehcache.constructs.web.filter.Filter.doFilter(Filter.java:97) at org .apache .catalina .core .ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java: 215) at org .apache .catalina .core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188) at org .appfuse .webapp.filter.LocaleFilter.doFilterInternal(LocaleFilter.java:74) at org.springframework.web.filter.OncePerRequestFilter.doFilter (OncePerRequestFilter.java:76) at org .apache .catalina .core .ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java: 215) at org .apache .catalina .core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188) at org .springframework .orm .hibernate3 .support .OpenSessionInViewFilter .doFilterInternal(OpenSessionInViewFilter.java:198) at org.springframework.web.filter.OncePerRequestFilter.doFilter (OncePerRequestFilter.java:76) at org .apache .catalina .core .ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java: 215) at org .apache .catalina .core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:359) at org.springframework.security.intercept.web.FilterSecurityInterceptor.invoke (FilterSecurityInterceptor.java:109) at org.springframework.security.intercept.web.FilterSecurityInterceptor.doFilter (FilterSecurityInterceptor.java:83) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .ui .ExceptionTranslationFilter .doFilterHttp(ExceptionTranslationFilter.java:101) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .providers .anonymous .AnonymousProcessingFilter .doFilterHttp(AnonymousProcessingFilter.java:105) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .ui .rememberme .RememberMeProcessingFilter .doFilterHttp(RememberMeProcessingFilter.java:116) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .wrapper .SecurityContextHolderAwareRequestFilter .doFilterHttp(SecurityContextHolderAwareRequestFilter.java:91) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .ui .basicauth .BasicProcessingFilter.doFilterHttp(BasicProcessingFilter.java:172) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .ui .AbstractProcessingFilter.doFilterHttp(AbstractProcessingFilter.java: 268) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilter.java:87) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .ui .SessionFixationProtectionFilter .doFilterHttp(SessionFixationProtectionFilter.java:61) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security .context .HttpSessionContextIntegrationFilter .doFilterHttp(HttpSessionContextIntegrationFilter.java:235) at org .springframework .security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java: 53) at org.springframework.security.util.FilterChainProxy $VirtualFilterChain.doFilter(FilterChainProxy.java:371) at org .springframework .security.util.FilterChainProxy.doFilter(FilterChainProxy.java:174) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate (DelegatingFilterProxy.java:236) at org.springframework.web.filter.DelegatingFilterProxy.doFilter (DelegatingFilterProxy.java:167) at org .apache .catalina .core .ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java: 215) at org .apache .catalina .core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188) at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal (CharacterEncodingFilter.java:96) at org.springframework.web.filter.OncePerRequestFilter.doFilter (OncePerRequestFilter.java:76) at org .apache .catalina .core .ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java: 215) at org .apache .catalina .core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188) at org .apache .catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java: 210) at org .apache .catalina.core.StandardContextValve.invoke(StandardContextValve.java: 174) at org .apache .catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) at org .apache .catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117) at org .apache .catalina.core.StandardEngineValve.invoke(StandardEngineValve.java: 108) at org .apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java: 151) at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:200) at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283) at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:773) at org .apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java: 703) at org.apache.jk.common.ChannelSocket $SocketConnection.runIt(ChannelSocket.java:895) at org.apache.tomcat.util.threads.ThreadPool $ControlRunnable.run(ThreadPool.java:685) at java.lang.Thread.run(Thread.java:595)
Andrea Hahn (ahahn@gbif.org) Data Portal Manager Global Biodiversity Information Facility Secretariat Universitetsparken 15, DK-2100 Copenhagen, Denmark Tel: +45-35321491 Fax: +45-35321480 URL: http://www.gbif.org
IPT mailing list IPT@lists.gbif.org http://lists.gbif.org/mailman/listinfo/ipt
participants (2)
-
"Markus Döring (GBIF)" -
Andrea Hahn (GBIF)