Merge remote-tracking branch 'upstream/main' into integ-IT-with-security

Signed-off-by: Yury-Fridlyand <[email protected]>
opensearch-project · Aug 17, 2023 · dd52ff1 · dd52ff1
2 parents f99d895 + bb3c340
commit dd52ff1
Show file tree

Hide file tree

Showing 360 changed files with 18,096 additions and 16,686 deletions.
diff --git a/build.gradle b/build.gradle
@@ -84,9 +84,16 @@ repositories {
 spotless {
     java {
         target fileTree('.') {
-            include 'common/**/*.java',
-                    'datasources/**/*.java',
-                    'core/**/*.java'
+            include 'datasources/**/*.java',
+                    'core/**/*.java',
+                    'protocol/**/*.java',
+                    'prometheus/**/*.java',
+                    'sql/**/*.java',
+                    'common/**/*.java',
+                    'spark/**/*.java',
+                    'plugin/**/*.java',
+                    'ppl/**/*.java',
+                    'integ-test/**/*java'
             exclude '**/build/**', '**/build-*/**'
         }
         importOrder()

diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java
@@ -161,7 +161,7 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
     table
         .getReservedFieldTypes()
-        .forEach((k, v) -> curEnv.addReservedWord(new Symbol(Namespace.FIELD_NAME, k), v));
+        .forEach((k, v) -> curEnv.define(new Symbol(Namespace.HIDDEN_FIELD_NAME, k), v));
 
     // Put index name or its alias in index namespace on type environment so qualifier
     // can be removed when analyzing qualified name. The value (expr type) here doesn't matter.
@@ -215,7 +215,7 @@ public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext contex
     table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v));
     table
         .getReservedFieldTypes()
-        .forEach((k, v) -> curEnv.addReservedWord(new Symbol(Namespace.FIELD_NAME, k), v));
+        .forEach((k, v) -> curEnv.define(new Symbol(Namespace.HIDDEN_FIELD_NAME, k), v));
     curEnv.define(
         new Symbol(
             Namespace.INDEX_NAME, dataSourceSchemaIdentifierNameResolver.getIdentifierName()),

diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java
@@ -378,7 +378,7 @@ public Expression visitQualifiedName(QualifiedName node, AnalysisContext context
           typeEnv != null;
           typeEnv = typeEnv.getParent()) {
         Optional<ExprType> exprType =
-            typeEnv.getReservedSymbolTable().lookup(new Symbol(Namespace.FIELD_NAME, part));
+            Optional.ofNullable(typeEnv.lookupAllFields(Namespace.HIDDEN_FIELD_NAME).get(part));
         if (exprType.isPresent()) {
           return visitMetadata(
               qualifierAnalyzer.unqualified(node), (ExprCoreType) exprType.get(), context);

diff --git a/core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java b/core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java
@@ -25,8 +25,6 @@ public class TypeEnvironment implements Environment<Symbol, ExprType> {
   @Getter private final TypeEnvironment parent;
   private final SymbolTable symbolTable;
 
-  @Getter private final SymbolTable reservedSymbolTable;
-
   /**
    * Constructor with empty symbol tables.
    *
@@ -35,7 +33,6 @@ public class TypeEnvironment implements Environment<Symbol, ExprType> {
   public TypeEnvironment(TypeEnvironment parent) {
     this.parent = parent;
     this.symbolTable = new SymbolTable();
-    this.reservedSymbolTable = new SymbolTable();
   }
 
   /**
@@ -47,7 +44,6 @@ public TypeEnvironment(TypeEnvironment parent) {
   public TypeEnvironment(TypeEnvironment parent, SymbolTable symbolTable) {
     this.parent = parent;
     this.symbolTable = symbolTable;
-    this.reservedSymbolTable = new SymbolTable();
   }
 
   /**
@@ -123,8 +119,4 @@ public void remove(ReferenceExpression ref) {
   public void clearAllFields() {
     lookupAllFields(FIELD_NAME).keySet().forEach(v -> remove(new Symbol(Namespace.FIELD_NAME, v)));
   }
-
-  public void addReservedWord(Symbol symbol, ExprType type) {
-    reservedSymbolTable.store(symbol, type);
-  }
 }
diff --git a/core/src/main/java/org/opensearch/sql/analysis/symbol/Namespace.java b/core/src/main/java/org/opensearch/sql/analysis/symbol/Namespace.java
@@ -9,6 +9,7 @@
 public enum Namespace {
   INDEX_NAME("Index"),
   FIELD_NAME("Field"),
+  HIDDEN_FIELD_NAME("HiddenField"),
   FUNCTION_NAME("Function");
 
   private final String name;

diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java
@@ -216,13 +216,13 @@ public void qualified_name_with_qualifier() {
   public void qualified_name_with_reserved_symbol() {
     analysisContext.push();
 
-    analysisContext.peek().addReservedWord(new Symbol(Namespace.FIELD_NAME, "_reserved"), STRING);
-    analysisContext.peek().addReservedWord(new Symbol(Namespace.FIELD_NAME, "_priority"), FLOAT);
+    analysisContext.peek().define(new Symbol(Namespace.HIDDEN_FIELD_NAME, "_reserved"), STRING);
+    analysisContext.peek().define(new Symbol(Namespace.HIDDEN_FIELD_NAME, "_priority"), FLOAT);
     analysisContext.peek().define(new Symbol(Namespace.INDEX_NAME, "index_alias"), STRUCT);
     assertAnalyzeEqual(DSL.ref("_priority", FLOAT), qualifiedName("_priority"));
     assertAnalyzeEqual(DSL.ref("_reserved", STRING), qualifiedName("index_alias", "_reserved"));
 
-    // reserved fields take priority over symbol table
+    // cannot replace an existing field type
     analysisContext.peek().define(new Symbol(Namespace.FIELD_NAME, "_reserved"), LONG);
     assertAnalyzeEqual(DSL.ref("_reserved", STRING), qualifiedName("index_alias", "_reserved"));
 

diff --git a/integ-test/build.gradle b/integ-test/build.gradle
@@ -56,6 +56,11 @@ repositories {
     }
 }
 
+// Being ignored as a temporary measure before being removed in favour of
+// spotless https://github.com/opensearch-project/sql/issues/1101
+checkstyleTest.ignoreFailures = true
+checkstyleMain.ignoreFailures = true
+
 ext {
     projectSubstitutions = [:]
     licenseFile = rootProject.file('LICENSE.TXT')