Added Schema reader, completion engine and a demo

pom.xml

@@ -23,7 +23,7 @@
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <encoding>UTF-8</encoding>
-    <scala.version>3.2.1</scala.version> <!-- Scala 2.12 not working because of copies of annotations issues -->
+    <scala.version>3.3.0</scala.version> <!-- Scala 2.12 not working because of copies of annotations issues -->
     <lsp4j.version>0.21.0</lsp4j.version>
     <typesafe.version>1.4.2</typesafe.version>
   </properties>
@@ -50,6 +50,13 @@
       <version>${typesafe.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>com.lihaoyi</groupId>
+      <artifactId>ujson_3</artifactId>
+      <version>3.1.2</version>
+    </dependency>
+
+
 
     <!-- Tests -->
     <dependency>

src/main/scala/io/smartdatalake/completion/SDLBCompletionEngine.scala

@@ -0,0 +1,7 @@
+package io.smartdatalake.completion
+
+import io.smartdatalake.context.SDLBContext
+import org.eclipse.lsp4j.CompletionItem
+
+trait SDLBCompletionEngine:
+  def generateCompletionItems(context: SDLBContext): List[CompletionItem]

src/main/scala/io/smartdatalake/completion/SDLBCompletionEngineImpl.scala

@@ -0,0 +1,37 @@
+package io.smartdatalake.completion
+
+import io.smartdatalake.completion.SDLBCompletionEngine
+import io.smartdatalake.completion.schema.{ItemType, SchemaItem, SchemaReader, SchemaReaderImpl}
+import io.smartdatalake.context.SDLBContext
+import org.eclipse.lsp4j.{CompletionItem, CompletionItemKind}
+
+import scala.util.{Failure, Success, Try}
+
+class SDLBCompletionEngineImpl extends SDLBCompletionEngine {
+
+  val schemaReader: SchemaReader = new SchemaReaderImpl("sdl-schema/sdl-schema-2.5.0.json") //TODO should be retrieved from a service keeping its state, object for example
+
+  override def generateCompletionItems(context: SDLBContext): List[CompletionItem] = context.parentPath match
+    case path if path.startsWith("actions") && path.count(_ == '.') == 1 => generatePropertiesOfAction(context)
+    case path if path.startsWith("actions") && !path.contains('.')  => List.empty[CompletionItem] //TODO discuss about this placeholder idea
+    case path if path.startsWith("actions") => List.empty[CompletionItem] //TODO when going deeper find a good recursive approach and mb merge it with first case
+    case _ => List.empty[CompletionItem]
+
+
+  private def generatePropertiesOfAction(context: SDLBContext): List[CompletionItem] =
+    val tActionType: Try[String] = Try(context.config.getString(context.parentPath + ".type"))
+    tActionType match
+      case Success(actionType) => schemaReader.retrieveActionProperties(actionType).map(createCompletionItem).toList
+      case Failure(_) => typeList
+
+  private def createCompletionItem(item: SchemaItem): CompletionItem =
+    val completionItem = new CompletionItem()
+    completionItem.setLabel(item.name)
+    completionItem.setDetail(item.description)
+    completionItem.setInsertText(item.name + (if item.itemType.isComplexValue then " " else " = "))
+    completionItem.setKind(CompletionItemKind.Snippet)
+    completionItem
+
+  private val typeItem = createCompletionItem(SchemaItem("type", ItemType.STRING, " type of object"))
+  private val typeList = List(typeItem)
+}

src/main/scala/io/smartdatalake/completion/schema/ItemType.scala

@@ -0,0 +1,22 @@
+package io.smartdatalake.completion.schema
+
+enum ItemType(val name: String) {
+  case STRING extends ItemType("string")
+  case BOOLEAN extends ItemType("boolean")
+  case INTEGER extends ItemType("integer")
+  case OBJECT extends ItemType("object")
+  case ARRAY extends ItemType("array")
+
+  def isPrimitiveValue: Boolean = this == ItemType.STRING || this == ItemType.BOOLEAN || this == ItemType.INTEGER
+
+  def isComplexValue: Boolean = this == ItemType.OBJECT || this == ItemType.ARRAY
+
+}
+
+object ItemType:
+  def fromName(name: String): ItemType = name match
+    case "string" => ItemType.STRING
+    case "boolean" => ItemType.BOOLEAN
+    case "integer" => ItemType.INTEGER
+    case "object" => ItemType.OBJECT
+    case "array" => ItemType.ARRAY

src/main/scala/io/smartdatalake/completion/schema/SchemaItem.scala

@@ -0,0 +1,3 @@
+package io.smartdatalake.completion.schema
+
+case class SchemaItem(name: String, itemType: ItemType, description: String) //TODO title as well?

src/main/scala/io/smartdatalake/completion/schema/SchemaReader.scala

@@ -0,0 +1,4 @@
+package io.smartdatalake.completion.schema
+
+trait SchemaReader:
+  def retrieveActionProperties(typeName: String): Iterable[SchemaItem]

src/main/scala/io/smartdatalake/completion/schema/SchemaReaderImpl.scala

@@ -0,0 +1,22 @@
+package io.smartdatalake.completion.schema
+
+import scala.io.Source
+import scala.util.Using
+
+class SchemaReaderImpl(val schemaPath: String) extends SchemaReader {
+
+  private val schema = ujson.read(Using.resource(getClass.getClassLoader.getResourceAsStream(schemaPath)) { inputStream =>
+    Source.fromInputStream(inputStream).getLines().mkString("\n").trim
+  })
+
+  override def retrieveActionProperties(typeName: String): Iterable[SchemaItem] =
+    val properties = schema("definitions")("Action")(typeName)("properties")
+
+    properties.obj.map { case (keyName, value) =>
+      val typeName = value.obj.get("type").map(_.str).getOrElse("string")
+      val description = value.obj.get("description").map(_.str).getOrElse("")
+      SchemaItem(keyName, ItemType.fromName(typeName), description)
+    }
+
+
+}

src/main/scala/io/smartdatalake/context/SDLBContext.scala

@@ -19,7 +19,7 @@ class SDLBContext private (val text: String, val config: Config, val parentPath:
    * get context of the parent
    * @return either a SimpleConfigObject if parent is a key or a ConfigString, ConfigList, ConfigBoolean etc if it is an end value
    */
-  def getParentContext: Option[ConfigValue] = if parentPath.isBlank then None else  Some(config.getValue(parentPath))
+  def getParentContext: Option[ConfigValue] = if parentPath.isBlank then None else Some(config.getValue(parentPath))
 
 }
 

src/main/scala/io/smartdatalake/context/hocon/HoconParser.scala

@@ -20,9 +20,7 @@ private[context] object HoconParser:
    * @return parsed text in config format
    */
   def parse(text: String): Option[Config] =
-    Try(ConfigFactory.parseString(text)) match
-      case Success(config) => Some(config)
-      case Failure(_) => None
+    Try(ConfigFactory.parseString(text)).toOption
 
   val EMPTY_CONFIG: Config = ConfigFactory.parseString("")
 

src/main/scala/io/smartdatalake/languageserver/SmartDataLakeTextDocumentService.scala

@@ -1,11 +1,15 @@
 package io.smartdatalake.languageserver
 
+import io.smartdatalake.completion.SDLBCompletionEngineImpl
+import io.smartdatalake.context.SDLBContext
 import org.eclipse.lsp4j.jsonrpc.messages
 import org.eclipse.lsp4j.services.TextDocumentService
 import org.eclipse.lsp4j.{CodeAction, CodeActionParams, CodeLens, CodeLensParams, Command, CompletionItem, CompletionItemKind, CompletionList, CompletionParams, DefinitionParams, DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams, DocumentFormattingParams, DocumentHighlight, DocumentHighlightParams, DocumentOnTypeFormattingParams, DocumentRangeFormattingParams, DocumentSymbol, DocumentSymbolParams, Hover, HoverParams, InsertReplaceEdit, Location, LocationLink, Position, Range, ReferenceParams, RenameParams, SignatureHelp, SignatureHelpParams, SymbolInformation, TextDocumentPositionParams, TextEdit, WorkspaceEdit}
 
 import java.util
 import java.util.concurrent.CompletableFuture
+import scala.io.Source
+import scala.util.Using
 
 class SmartDataLakeTextDocumentService extends TextDocumentService {
 
@@ -14,31 +18,45 @@ class SmartDataLakeTextDocumentService extends TextDocumentService {
     CompletableFuture.supplyAsync(() => {
       val completionItems = new util.ArrayList[CompletionItem]()
 
-      if (params.getPosition.getLine == 1) {
-        val completionItem = new CompletionItem()
-        completionItem.setInsertText("dataObjects {\n\t\n}\n\nactions {\n\t\n}")
-        completionItem.setLabel("gen")
-        completionItem.setKind(CompletionItemKind.Snippet)
-        completionItem.setDetail("Generate basic template")
-
-        completionItems.add(completionItem)
-      } else {
-        // Sample Completion item for dataObject
-        val completionItem = new CompletionItem()
-        // Define the text to be inserted in to the file if the completion item is selected.
-        completionItem.setInsertText("dataObjects")
-        // Set the label that shows when the completion drop down appears in the Editor.
-        completionItem.setLabel("dataObjects")
-        // Set the completion kind. This is a snippet.
-        // That means it replace character which trigger the completion and
-        // replace it with what defined in inserted text.
-        completionItem.setKind(CompletionItemKind.Snippet)
-        // This will set the details for the snippet code which will help user to
-        // understand what this completion item is.
-        completionItem.setDetail(" {...}\n Defines the data objects")
-        // Add the sample completion item to the list.
-        completionItems.add(completionItem)
-      }
+      val fixtureText = //TODO weird behavior with \"\"\"
+        """actions {
+          |
+          |  join-departures-airports {
+          |    type = CustomDataFrameAction
+          |
+          |    inputIds = [stg-departures, int-airports]
+          |    transformer = {
+          |      type = SQLDfsTransformer
+          |      code = {
+          |        btl-connected-airports = "select stg_departures.estdepartureairport, stg_departures.estarrivalairport,        airports.*         from stg_departures join int_airports airports on stg_departures.estArrivalAirport = airports.ident"
+          |      }
+          |    }
+          |  }
+          |
+          |  compute-distances {
+          |    type = CopyAction
+          |
+          |    code = {
+          |      btl-departures-arrivals-airports = "select btl_connected_airports.estdepartureairport, btl_connected_airports.estarrivalairport,        btl_connected_airports.name as arr_name, btl_connected_airports.latitude_deg as arr_latitude_deg, btl_connected_airports.longitude_deg as arr_longitude_deg,        airports.name as dep_name, airports.latitude_deg as dep_latitude_deg, airports.longitude_deg as dep_longitude_deg           from btl_connected_airports join int_airports airports on btl_connected_airports.estdepartureairport = airports.ident"
+          |    }
+          |    metadata {
+          |      feed = compute
+          |    }
+          |  }
+          |
+          |  download-airports  {
+          |
+          |    inputId = ext-airports
+          |  }
+          |
+          |}
+          |
+          |dataObjects {
+          |
+          |
+          |}""".stripMargin.trim
+      val suggestions: List[CompletionItem] = new SDLBCompletionEngineImpl().generateCompletionItems(SDLBContext.createContext(fixtureText, params.getPosition.getLine+1, params.getPosition.getCharacter))
+      suggestions.foreach(e => completionItems.add(e))
 
       messages.Either.forLeft(completionItems).asInstanceOf[messages.Either[util.List[CompletionItem], CompletionList]]
     })

src/main/scala/io/smartdatalake/utils/MultiLineTransformer.scala

@@ -15,11 +15,15 @@ object MultiLineTransformer {
 
 
   def computeCorrectedPositions(text: String): List[(Int, Int)] =
+
+    def isMultilineModeStartingOrEnding(line: String): Boolean =
+      // handle specific case where the starting """ and ending """ are in the same line or not.
+      line.count(_ == '"') % 2 == 1
     case class State(isInMultiLine: Boolean, lineNumber: Int, columnShift: Int)
     text.split("\n")
       .foldLeft(List(State(false, 1, 0))) {(states, line) =>
         val lastState = states.head
-        val isInTripleQuotes = lastState.isInMultiLine ^ line.contains("\"\"\"")
+        val isInTripleQuotes = lastState.isInMultiLine ^ isMultilineModeStartingOrEnding(line)
         if isInTripleQuotes then
           State(isInTripleQuotes, lastState.lineNumber, lastState.columnShift + line.length)::states
         else