#16 increased content buffer size

gbif · Aug 3, 2023 · e466113 · e466113
1 parent 39b303d
commit e466113
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 3 deletions.
diff --git a/src/main/java/org/gbif/literature/export/LiteraturePager.java b/src/main/java/org/gbif/literature/export/LiteraturePager.java
@@ -40,6 +40,6 @@ public LiteraturePager(
   public PagingResponse<LiteratureSearchResult> nextPage(PagingRequest pagingRequest) {
     literatureSearchRequest.setOffset(pagingRequest.getOffset());
     literatureSearchRequest.setLimit(pagingRequest.getLimit());
-    return literatureSearchService.search(literatureSearchRequest);
+    return literatureSearchService.exportSearch(literatureSearchRequest);
   }
 }
diff --git a/src/main/java/org/gbif/literature/search/LiteratureSearchService.java b/src/main/java/org/gbif/literature/search/LiteratureSearchService.java
@@ -13,6 +13,7 @@
  */
 package org.gbif.literature.search;
 
+import org.gbif.api.model.common.search.SearchResponse;
 import org.gbif.api.model.literature.search.LiteratureSearchParameter;
 import org.gbif.api.model.literature.search.LiteratureSearchRequest;
 import org.gbif.api.model.literature.search.LiteratureSearchResult;
@@ -25,4 +26,7 @@ public interface LiteratureSearchService
         LiteratureSearchResult, LiteratureSearchParameter, LiteratureSearchRequest> {
 
   Optional<LiteratureSearchResult> get(Object identifier);
+
+  SearchResponse<LiteratureSearchResult, LiteratureSearchParameter> exportSearch(
+      LiteratureSearchRequest literatureSearchRequest);
 }
diff --git a/src/main/java/org/gbif/literature/search/LiteratureSearchServiceEs.java b/src/main/java/org/gbif/literature/search/LiteratureSearchServiceEs.java
@@ -23,13 +23,15 @@
 import java.util.Optional;
 
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.springframework.stereotype.Service;
 
 @Service
 public class LiteratureSearchServiceEs implements LiteratureSearchService {
 
+  private static final int BUFFER_LIMIT_BYTES_EXPORT = 140_000_000;
   private final RestHighLevelClient restHighLevelClient;
   private final LiteratureEsResponseParser esResponseParser;
   private final EsSearchRequestBuilder<LiteratureSearchParameter> esSearchRequestBuilder;
@@ -51,6 +53,11 @@ public LiteratureSearchServiceEs(
   @Override
   public SearchResponse<LiteratureSearchResult, LiteratureSearchParameter> search(
       LiteratureSearchRequest literatureSearchRequest) {
+    return searchInternal(literatureSearchRequest, RequestOptions.DEFAULT);
+  }
+
+  private SearchResponse<LiteratureSearchResult, LiteratureSearchParameter> searchInternal(
+      LiteratureSearchRequest literatureSearchRequest, RequestOptions requestOptions) {
     int limit = literatureSearchRequest.getLimit();
     long offset = literatureSearchRequest.getOffset();
     boolean offsetExceeded = false;
@@ -65,8 +72,7 @@ public SearchResponse<LiteratureSearchResult, LiteratureSearchParameter> search(
           esSearchRequestBuilder.buildSearchRequest(literatureSearchRequest, true, index);
       SearchResponse<LiteratureSearchResult, LiteratureSearchParameter> response =
           esResponseParser.buildSearchResponse(
-              restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT),
-              literatureSearchRequest);
+              restHighLevelClient.search(searchRequest, requestOptions), literatureSearchRequest);
 
       if (offsetExceeded) {
         response.setOffset(offset);
@@ -88,4 +94,14 @@ public Optional<LiteratureSearchResult> get(Object identifier) {
       throw new RuntimeException(e);
     }
   }
+
+  @Override
+  public SearchResponse<LiteratureSearchResult, LiteratureSearchParameter> exportSearch(
+      LiteratureSearchRequest literatureSearchRequest) {
+    RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+    builder.setHttpAsyncResponseConsumerFactory(
+        new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(
+            BUFFER_LIMIT_BYTES_EXPORT));
+    return searchInternal(literatureSearchRequest, builder.build());
+  }
 }