Add new setting to ignore pagination where it is unsupported in V2.

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -32,6 +32,7 @@ public enum Key {
     QUERY_SIZE_LIMIT("plugins.query.size_limit"),
     ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"),
     DATASOURCES_URI_ALLOWHOSTS("plugins.query.datasources.uri.allowhosts"),
+    IGNORE_UNSUPPORTED_PAGINATION("plugins.query.ignore_unsupported_pagination"),
 
     METRICS_ROLLING_WINDOW("plugins.query.metrics.rolling_window"),
     METRICS_ROLLING_INTERVAL("plugins.query.metrics.rolling_interval"),

core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java

@@ -13,6 +13,8 @@
 import java.util.Optional;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.statement.Explain;
 import org.opensearch.sql.ast.statement.Query;
@@ -21,6 +23,8 @@
 import org.opensearch.sql.ast.tree.FetchCursor;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.common.response.ResponseListener;
+import org.opensearch.sql.common.setting.Settings;
+import org.opensearch.sql.common.utils.QueryContext;
 import org.opensearch.sql.exception.UnsupportedCursorRequestException;
 import org.opensearch.sql.executor.ExecutionEngine;
 import org.opensearch.sql.executor.QueryId;
@@ -39,6 +43,10 @@ public class QueryPlanFactory
   /** Query Service. */
   private final QueryService queryService;
 
+  private final Settings settings;
+
+  private static final Logger LOG = LogManager.getLogger(QueryPlanFactory.class);
+
   /**
    * NO_CONSUMER_RESPONSE_LISTENER should never be called. It is only used as constructor parameter
    * of {@link QueryPlan}.
@@ -111,6 +119,10 @@ public AbstractPlan visitQuery(
             node.getFetchSize(),
             queryService,
             context.getLeft().get());
+      } else if (settings.getSettingValue(Settings.Key.IGNORE_UNSUPPORTED_PAGINATION)) {
+        LOG.warn("[{}] Query executed without pagination.", QueryContext.getRequestId());
+        return new QueryPlan(
+            QueryId.queryId(), node.getPlan(), queryService, context.getLeft().get());
       } else {
         // This should be picked up by the legacy engine.
         throw new UnsupportedCursorRequestException();

core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java

@@ -10,6 +10,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -32,6 +33,7 @@
 import org.opensearch.sql.ast.tree.CloseCursor;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.common.response.ResponseListener;
+import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.exception.UnsupportedCursorRequestException;
 import org.opensearch.sql.executor.ExecutionEngine;
 import org.opensearch.sql.executor.QueryService;
@@ -51,11 +53,13 @@ class QueryPlanFactoryTest {
 
   @Mock private ExecutionEngine.QueryResponse queryResponse;
 
+  @Mock private Settings settings;
+
   private QueryPlanFactory factory;
 
   @BeforeEach
   void init() {
-    factory = new QueryPlanFactory(queryService);
+    factory = new QueryPlanFactory(queryService, settings);
   }
 
   @Test
@@ -125,26 +129,40 @@ public void no_consumer_response_channel() {
   @Test
   public void create_query_with_fetch_size_which_can_be_paged() {
     when(plan.accept(any(CanPaginateVisitor.class), any())).thenReturn(Boolean.TRUE);
-    factory = new QueryPlanFactory(queryService);
+    factory = new QueryPlanFactory(queryService, settings);
     Statement query = new Query(plan, 10);
     AbstractPlan queryExecution =
         factory.create(query, Optional.of(queryListener), Optional.empty());
     assertTrue(queryExecution instanceof QueryPlan);
+    assertEquals(10, ((QueryPlan) queryExecution).pageSize.get());
   }
 
   @Test
-  public void create_query_with_fetch_size_which_cannot_be_paged() {
+  public void create_query_with_fetch_size_which_cannot_be_paged_with_legacy_fallback() {
     when(plan.accept(any(CanPaginateVisitor.class), any())).thenReturn(Boolean.FALSE);
-    factory = new QueryPlanFactory(queryService);
+    when(settings.getSettingValue(Settings.Key.IGNORE_UNSUPPORTED_PAGINATION)).thenReturn(false);
+    factory = new QueryPlanFactory(queryService, settings);
     Statement query = new Query(plan, 10);
     assertThrows(
         UnsupportedCursorRequestException.class,
         () -> factory.create(query, Optional.of(queryListener), Optional.empty()));
   }
 
+  @Test
+  public void create_query_with_fetch_size_which_cannot_be_paged_with_ignoring_paging() {
+    when(plan.accept(any(CanPaginateVisitor.class), any())).thenReturn(Boolean.FALSE);
+    when(settings.getSettingValue(Settings.Key.IGNORE_UNSUPPORTED_PAGINATION)).thenReturn(true);
+    factory = new QueryPlanFactory(queryService, settings);
+    Statement query = new Query(plan, 10);
+    AbstractPlan queryExecution =
+        factory.create(query, Optional.of(queryListener), Optional.empty());
+    assertTrue(queryExecution instanceof QueryPlan);
+    assertFalse(((QueryPlan) queryExecution).pageSize.isPresent());
+  }
+
   @Test
   public void create_close_cursor() {
-    factory = new QueryPlanFactory(queryService);
+    factory = new QueryPlanFactory(queryService, settings);
     var plan = factory.createCloseCursor("pewpew", queryListener);
     assertTrue(plan instanceof CommandPlan);
     plan.execute();

docs/user/admin/settings.rst

@@ -36,7 +36,7 @@ The opendistro.sql.query.analysis.semantic.threshold setting is deprecated and w
 
 opendistro.sql.query.response.format
 ------------------------------------
-The opendistro.sql.query.response.format setting is deprecated and will be removed then. From OpenSearch 1.0, the query response format is default to JDBC format. `You can change the format by using query parameters<../interfaces/protocol.rst>`_.
+The opendistro.sql.query.response.format setting is deprecated and will be removed then. From OpenSearch 1.0, the query response format is default to JDBC format. `You can change the format by using query parameters <../interfaces/protocol.rst>`_.
 
 opendistro.sql.cursor.enabled
 -----------------------------
@@ -47,7 +47,7 @@ opendistro.sql.cursor.fetch_size
 The opendistro.sql.cursor.fetch_size setting is deprecated and will be removed then. From OpenSearch 1.0, the fetch_size in query body will decide whether create the cursor context. No cursor will be created if the fetch_size = 0.
 
 plugins.sql.enabled
-======================
+===================
 
 Description
 -----------
@@ -64,15 +64,15 @@ Example 1
 
 You can update the setting with a new value like this.
 
-SQL query::
+Configuration request::
 
 	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{
 	  "transient" : {
 	    "plugins.sql.enabled" : "false"
 	  }
 	}'
 
-Result set::
+Response::
 
 	{
 	  "acknowledged" : true,
@@ -99,7 +99,7 @@ SQL query::
 	  "query" : "SELECT * FROM accounts"
 	}'
 
-Result set::
+Response::
 
 	{
 	  "error" : {
@@ -111,7 +111,7 @@ Result set::
 	}
 
 plugins.sql.slowlog
-============================
+===================
 
 Description
 -----------
@@ -128,15 +128,15 @@ Example
 
 You can update the setting with a new value like this.
 
-SQL query::
+Configuration request::
 
 	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{
 	  "transient" : {
 	    "plugins.query.slowlog" : "10"
 	  }
 	}'
 
-Result set::
+Response::
 
 	{
 	  "acknowledged" : true,
@@ -153,7 +153,7 @@ Result set::
 Note: the legacy settings of ``opendistro.sql.slowlog`` is deprecated, it will fallback to the new settings if you request an update with the legacy name.
 
 plugins.sql.cursor.keep_alive
-================================
+=============================
 
 Description
 -----------
@@ -170,15 +170,15 @@ Example
 
 You can update the setting with a new value like this.
 
-SQL query::
+Configuration request::
 
 	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{
 	  "transient" : {
 	    "plugins.sql.cursor.keep_alive" : "5m"
 	  }
 	}'
 
-Result set::
+Response::
 
 	{
 	  "acknowledged" : true,
@@ -197,7 +197,7 @@ Result set::
 Note: the legacy settings of ``opendistro.sql.cursor.keep_alive`` is deprecated, it will fallback to the new settings if you request an update with the legacy name.
 
 plugins.query.size_limit
-===========================
+========================
 
 Description
 -----------
@@ -210,7 +210,7 @@ The new engine fetches a default size of index from OpenSearch set by this setti
 	  }
 	}'
 
-Result set::
+Response::
 
     {
       "acknowledged" : true,
@@ -240,25 +240,25 @@ You can set heap memory usage limit for the query engine. When query running, it
 	  }
 	}'
 
-Result set::
+Response::
 
     {
       "acknowledged": true,
-      "persistent": {
+      "persistent": { },
+      "transient": {
         "plugins": {
           "query": {
             "memory_limit": "80%"
           }
         }
-      },
-      "transient": {}
+      }
     }
 
 Note: the legacy settings of ``opendistro.ppl.query.memory_limit`` is deprecated, it will fallback to the new settings if you request an update with the legacy name.
 
 
 plugins.sql.delete.enabled
-======================
+==========================
 
 Description
 -----------
@@ -275,10 +275,10 @@ Example 1
 
 You can update the setting with a new value like this.
 
-SQL query::
+Configuration request::
 
     sh$ curl -sS -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings \
-    ... -d '{"transient":{"plugins.sql.delete.enabled":"false"}}'
+    ... -d '{ "transient" : { "plugins.sql.delete.enabled" : false } }'
     {
       "acknowledged": true,
       "persistent": {},
@@ -311,3 +311,38 @@ SQL query::
       "status": 400
     }
 
+plugins.query.ignore_unsupported_pagination
+===========================================
+
+Description
+-----------
+
+This boolean settings defines how SQL plugin handles pagination requests with features which are not supported or not compatible with pagination in V2 engine.
+
+Please, refer to `V1/V2 limitations <https://opensearch.org/docs/latest/search-plugins/sql/limitation/#query-processing-engines>`_ and `pagination request <../interfaces/endpoint.rst#cursor>`_ info sections.
+
+1. This setting is set to ``true`` by default.
+2. When it is set to ``true``, such requests are performed without pagination.
+3. When it is set to ``false``, such requests are performed by V1 engine.
+
+Configuration request::
+
+	>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{
+	  "transient" : {
+	    "plugins.query.ignore_unsupported_pagination" : false
+	  }
+	}'
+
+Response::
+
+    {
+      "acknowledged": true,
+      "persistent": { },
+      "transient": {
+        "plugins": {
+          "query": {
+            "ignore_unsupported_pagination": "false"
+          }
+        }
+      }
+    }

docs/user/index.rst

@@ -3,7 +3,7 @@
 OpenSearch SQL Reference Manual
 ===============================
 
-OpenSearch SQL enables you to extract insights out of OpenSearch using the familiar SQL query syntax. Please refer to the `technical documentation <https://docs-beta.opensearch.org/>`_ for detailed information on installing and configuring opendistro-elasticsearch-sql plugin. In this user reference manual, you can find many information for your reference. In each part, we try to make it clear by adding work example along with detailed description. Here is table of contents of the documentation:
+OpenSearch SQL enables you to extract insights out of OpenSearch using the familiar SQL query syntax. Please refer to the `technical documentation <https://opensearch.org/docs/latest/install-and-configure/plugins/>`_ for detailed information on installing and configuring SQL plugin. In this user reference manual, you can find many information for your reference. In each part, we try to make it clear by adding work example along with detailed description. Here is table of contents of the documentation:
 
 * **Interfaces**
 

integ-test/src/test/java/org/opensearch/sql/correctness/runner/connection/JDBCConnection.java

@@ -108,7 +108,9 @@ public void insert(String tableName, String[] columnNames, List<Object[]> batch)
 
   @Override
   public DBResult select(String query) {
-    try (Statement stmt = connection.createStatement()) {
+    try (Statement stmt =
+        connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) {
+      stmt.setFetchSize(5);
       ResultSet resultSet = stmt.executeQuery(query);
       DBResult result =
           isOrderByQuery(query)

integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java

@@ -39,6 +39,9 @@ public class CursorIT extends SQLIntegTestCase {
   @Override
   protected void init() throws Exception {
     loadIndex(Index.ACCOUNT);
+    updateClusterSettings(
+        new ClusterSetting(
+            PERSISTENT, Settings.Key.IGNORE_UNSUPPORTED_PAGINATION.getKeyValue(), "false"));
   }
 
   /**

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -392,7 +392,7 @@ protected static JSONObject updateClusterSettings(ClusterSetting setting, RestCl
     return new JSONObject(executeRequest(request, client));
   }
 
-  protected static JSONObject updateClusterSettings(ClusterSetting setting) throws IOException {
+  public static JSONObject updateClusterSettings(ClusterSetting setting) throws IOException {
     return updateClusterSettings(setting, client());
   }
 
@@ -404,7 +404,7 @@ protected static JSONObject getAllClusterSettings() throws IOException {
     return new JSONObject(executeRequest(request));
   }
 
-  protected static class ClusterSetting {
+  public static class ClusterSetting {
     private final String type;
     private final String name;
     private final String value;

integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java

@@ -240,7 +240,7 @@ public QueryPlanFactory queryPlanFactory(ExecutionEngine executionEngine) {
               new ExpressionAnalyzer(functionRepository), dataSourceService, functionRepository);
       Planner planner = new Planner(LogicalPlanOptimizer.create());
       QueryService queryService = new QueryService(analyzer, executionEngine, planner);
-      return new QueryPlanFactory(queryService);
+      return new QueryPlanFactory(queryService, settings);
     }
   }