diff --git a/src/main/java/org/gbif/literature/export/LiteraturePager.java b/src/main/java/org/gbif/literature/export/LiteraturePager.java index dab7bec..e757f1f 100644 --- a/src/main/java/org/gbif/literature/export/LiteraturePager.java +++ b/src/main/java/org/gbif/literature/export/LiteraturePager.java @@ -40,6 +40,6 @@ public LiteraturePager( public PagingResponse nextPage(PagingRequest pagingRequest) { literatureSearchRequest.setOffset(pagingRequest.getOffset()); literatureSearchRequest.setLimit(pagingRequest.getLimit()); - return literatureSearchService.search(literatureSearchRequest); + return literatureSearchService.exportSearch(literatureSearchRequest); } } diff --git a/src/main/java/org/gbif/literature/search/LiteratureSearchService.java b/src/main/java/org/gbif/literature/search/LiteratureSearchService.java index 261d622..3c113ec 100644 --- a/src/main/java/org/gbif/literature/search/LiteratureSearchService.java +++ b/src/main/java/org/gbif/literature/search/LiteratureSearchService.java @@ -13,6 +13,7 @@ */ package org.gbif.literature.search; +import org.gbif.api.model.common.search.SearchResponse; import org.gbif.api.model.literature.search.LiteratureSearchParameter; import org.gbif.api.model.literature.search.LiteratureSearchRequest; import org.gbif.api.model.literature.search.LiteratureSearchResult; @@ -25,4 +26,7 @@ public interface LiteratureSearchService LiteratureSearchResult, LiteratureSearchParameter, LiteratureSearchRequest> { Optional get(Object identifier); + + SearchResponse exportSearch( + LiteratureSearchRequest literatureSearchRequest); } diff --git a/src/main/java/org/gbif/literature/search/LiteratureSearchServiceEs.java b/src/main/java/org/gbif/literature/search/LiteratureSearchServiceEs.java index 5f29582..d93f0ea 100644 --- a/src/main/java/org/gbif/literature/search/LiteratureSearchServiceEs.java +++ b/src/main/java/org/gbif/literature/search/LiteratureSearchServiceEs.java @@ -23,6 +23,7 @@ import java.util.Optional; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.stereotype.Service; @@ -30,6 +31,7 @@ @Service public class LiteratureSearchServiceEs implements LiteratureSearchService { + private static final int BUFFER_LIMIT_BYTES_EXPORT = 140_000_000; private final RestHighLevelClient restHighLevelClient; private final LiteratureEsResponseParser esResponseParser; private final EsSearchRequestBuilder esSearchRequestBuilder; @@ -51,6 +53,11 @@ public LiteratureSearchServiceEs( @Override public SearchResponse search( LiteratureSearchRequest literatureSearchRequest) { + return searchInternal(literatureSearchRequest, RequestOptions.DEFAULT); + } + + private SearchResponse searchInternal( + LiteratureSearchRequest literatureSearchRequest, RequestOptions requestOptions) { int limit = literatureSearchRequest.getLimit(); long offset = literatureSearchRequest.getOffset(); boolean offsetExceeded = false; @@ -65,8 +72,7 @@ public SearchResponse search( esSearchRequestBuilder.buildSearchRequest(literatureSearchRequest, true, index); SearchResponse response = esResponseParser.buildSearchResponse( - restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT), - literatureSearchRequest); + restHighLevelClient.search(searchRequest, requestOptions), literatureSearchRequest); if (offsetExceeded) { response.setOffset(offset); @@ -88,4 +94,14 @@ public Optional get(Object identifier) { throw new RuntimeException(e); } } + + @Override + public SearchResponse exportSearch( + LiteratureSearchRequest literatureSearchRequest) { + RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setHttpAsyncResponseConsumerFactory( + new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory( + BUFFER_LIMIT_BYTES_EXPORT)); + return searchInternal(literatureSearchRequest, builder.build()); + } }