[IPT] text file upload (larger record numbers)

[Andrea Hahn (GBIF)]

Hi,

 

quick note: I have been playing a bit with text file uploads for occurrence
records, using a tabular export from the GBIF index on a 500,000+ record
dataset. Using different subsets of the same recordset, I have not been able
to get beyond the 310,000 records cutdown of the test set so far - this
equals a file size of 99,974kB in the given example. The next larger set
(320,000 records, 100MB) and all larger ones fail with 


 Yikes!


javax.servlet.ServletException: Expression resource is undefined on line 87,
column 30 in WEB-INF/pages/manage/sources.ftl.

        at
net.sf.ehcache.constructs.web.filter.Filter.logThrowable(Filter.java:152)

        at
net.sf.ehcache.constructs.web.filter.Filter.doFilter(Filter.java:97)

        at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application
FilterChain.java:215)

        at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh
ain.java:188)

        at
org.appfuse.webapp.filter.LocaleFilter.doFilterInternal(LocaleFilter.java:74
)

        at
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestF
ilter.java:76)

        at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application
FilterChain.java:215)

        at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh
ain.java:188)

        at
org.springframework.orm.hibernate3.support.OpenSessionInViewFilter.doFilterI
nternal(OpenSessionInViewFilter.java:198)

        at
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestF
ilter.java:76)

        at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application
FilterChain.java:215)

        at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh
ain.java:188)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:359)

        at
org.springframework.security.intercept.web.FilterSecurityInterceptor.invoke(
FilterSecurityInterceptor.java:109)

        at
org.springframework.security.intercept.web.FilterSecurityInterceptor.doFilte
r(FilterSecurityInterceptor.java:83)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.ui.ExceptionTranslationFilter.doFilterHttp(Exce
ptionTranslationFilter.java:101)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.providers.anonymous.AnonymousProcessingFilter.d
oFilterHttp(AnonymousProcessingFilter.java:105)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.ui.rememberme.RememberMeProcessingFilter.doFilt
erHttp(RememberMeProcessingFilter.java:116)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.wrapper.SecurityContextHolderAwareRequestFilter
.doFilterHttp(SecurityContextHolderAwareRequestFilter.java:91)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.ui.basicauth.BasicProcessingFilter.doFilterHttp
(BasicProcessingFilter.java:172)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.ui.AbstractProcessingFilter.doFilterHttp(Abstra
ctProcessingFilter.java:268)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilte
r.java:87)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.ui.SessionFixationProtectionFilter.doFilterHttp
(SessionFixationProtectionFilter.java:61)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.context.HttpSessionContextIntegrationFilter.doF
ilterHttp(HttpSessionContextIntegrationFilter.java:235)

        at
org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurity
Filter.java:53)

        at
org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilt
er(FilterChainProxy.java:371)

        at
org.springframework.security.util.FilterChainProxy.doFilter(FilterChainProxy
.java:174)

        at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(Delegati
ngFilterProxy.java:236)

        at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilt
erProxy.java:167)

        at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application
FilterChain.java:215)

        at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh
ain.java:188)

        at
org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(Char
acterEncodingFilter.java:96)

        at
org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestF
ilter.java:76)

        at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Application
FilterChain.java:215)

        at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterCh
ain.java:188)

        at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.ja
va:210)

        at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.ja
va:174)

        at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127
)

        at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117
)

        at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java
:108)

        at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:151)

        at
org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:200)

        at
org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)

        at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:773)

        at
org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:703)

        at
org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java
:895)

        at
org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.jav
a:685)

        at java.lang.Thread.run(Thread.java:595)

 

 

---------------------------------------------------------------

Andrea Hahn  (ahahn at gbif.org)

Data Portal Manager

Global Biodiversity Information Facility Secretariat 

Universitetsparken 15, DK-2100 Copenhagen, Denmark

Tel: +45-35321491 Fax: +45-35321480

URL: http://www.gbif.org

---------------------------------------------------------------

 

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.gbif.org/pipermail/ipt/attachments/20090227/65472a90/attachment-0001.html