Added reading schema capabilities to full general purpose. Improved l…

…ist accuracy. Added intelligence between actions and dataObjects. but carefull, MISSING TESTS now

src/main/scala/io/smartdatalake/completion/SDLBCompletionEngineImpl.scala

@@ -1,29 +1,51 @@
 package io.smartdatalake.completion
 
+import com.typesafe.config.{Config, ConfigList, ConfigObject, ConfigValue}
 import io.smartdatalake.completion.SDLBCompletionEngine
-import io.smartdatalake.context.SDLBContext
+import io.smartdatalake.context.{SDLBContext, TextContext}
+import io.smartdatalake.schema.SchemaCollections.{AttributeCollection, TemplateCollection}
 import io.smartdatalake.schema.{ItemType, SchemaItem, SchemaReader, SchemaReaderImpl}
+import io.smartdatalake.conversions.ScalaJavaConverterAPI.*
 import org.eclipse.lsp4j.{CompletionItem, CompletionItemKind}
 
 import scala.util.{Failure, Success, Try}
 
 class SDLBCompletionEngineImpl(private val schemaReader: SchemaReader) extends SDLBCompletionEngine {
 
-  override def generateCompletionItems(context: SDLBContext): List[CompletionItem] = context.parentPath match //TODO split path by '.' and handle logic if element list or not like that? Carry over the new config like that too?
-    case path if path.startsWith("actions") && path.count(_ == '.') == 1 => generatePropertiesOfAction(context)
-    case "actions"  => generateTemplatesForAction()
-    case path if path.startsWith("actions") => List.empty[CompletionItem] //TODO when going deeper find a good recursive approach and mb merge it with first case
-    case _ => List.empty[CompletionItem]
+  override def generateCompletionItems(context: SDLBContext): List[CompletionItem] =
+    val itemSuggestionsFromSchema = schemaReader.retrieveAttributeOrTemplateCollection(context) match
+      case AttributeCollection(attributes) => generateAttributeSuggestions(attributes, context.getParentContext)
+      case TemplateCollection(templates) => generateTemplateSuggestions(templates, context.isInList)
 
+    val itemSuggestionsFromConfig = generateItemSuggestionsFromConfig(context)
+    val allItems = itemSuggestionsFromConfig ++ itemSuggestionsFromSchema
+    if allItems.isEmpty then typeList else allItems
 
-  private[completion] def generateTemplatesForAction(): List[CompletionItem] =
-    val actionsWithRequiredAttr = schemaReader.retrieveActionTypesWithRequiredAttributes()
-    actionsWithRequiredAttr.map { case (actionType, attributes) =>
+  private def generateItemSuggestionsFromConfig(context: SDLBContext): List[CompletionItem] = context.parentPath.lastOption match
+    case Some(value) => value match
+      case "inputId" | "outputId" => retrieveDataObjectIds(context)
+      case _ => List.empty[CompletionItem]
+    case None => List.empty[CompletionItem]
+
+  private def retrieveDataObjectIds(context: SDLBContext): List[CompletionItem] = //TODO test
+    context.textContext.rootConfig.getValue("dataObjects") match
+      case asConfigObject: ConfigObject => asConfigObject.unwrapped().keySet().toScala.toList.map(createCompletionItem)
+
+      case _ => List.empty[CompletionItem]
+  private def generateAttributeSuggestions(attributes: Iterable[SchemaItem], parentContext: Option[ConfigValue]): List[CompletionItem] =
+    val items = parentContext match
+      case Some(config: ConfigObject) => attributes.filter(item => Option(config.get(item.name)).isEmpty)
+      case _ => attributes
+    items.map(createCompletionItem).toList
+
+  private[completion] def generateTemplateSuggestions(templates: Iterable[(String, Iterable[SchemaItem])], isInList: Boolean): List[CompletionItem] =
+    templates.map { case (actionType, attributes) =>
       val completionItem = new CompletionItem()
       completionItem.setLabel(actionType.toLowerCase)
       completionItem.setDetail("  template")
+      val keyName = if isInList then "" else s"${actionType.toLowerCase}_PLACEHOLDER"
       completionItem.setInsertText(
-        s"""${actionType.toLowerCase}_PLACEHOLDER {
+        s"""$keyName {
           |${
           def generatePlaceHolderValue(att: SchemaItem) = {
             if att.name == "type" then actionType else att.itemType.defaultValue
@@ -34,19 +56,19 @@ class SDLBCompletionEngineImpl(private val schemaReader: SchemaReader) extends S
       completionItem
     }.toList
 
-
-  private def generatePropertiesOfAction(context: SDLBContext): List[CompletionItem] =
-    def isMissingInConfig(item: SchemaItem): Boolean = Try(context.textContext.config.getAnyRef(context.parentPath + "." + item.name)).isFailure
-    val tActionType: Try[String] = Try(context.textContext.config.getString(context.parentPath + ".type")) // In list, it looks like fixture.config.getList("actions.select-airport-cols.transformers").get(0).asInstanceOf[ConfigObject].get("type").unwrapped()
-    tActionType match
-      case Success(actionType) => schemaReader.retrieveActionProperties(actionType).filter(isMissingInConfig).map(createCompletionItem).toList
-      case Failure(_) => typeList
-
   private def createCompletionItem(item: SchemaItem): CompletionItem =
     val completionItem = new CompletionItem()
     completionItem.setLabel(item.name)
     completionItem.setDetail(f"  ${if item.required then "required" else ""}%s ${item.itemType.name}%-10s") //TODO check how to justify properly
-    completionItem.setInsertText(item.name + (if item.itemType.isComplexValue then " " else " = "))
+    completionItem.setInsertText(item.name + (if item.itemType == ItemType.OBJECT then " " else " = ") + item.itemType.defaultValue)
+    completionItem.setKind(CompletionItemKind.Snippet)
+    completionItem
+
+  private def createCompletionItem(item: String): CompletionItem =
+    val completionItem = new CompletionItem()
+    completionItem.setLabel(item)
+    completionItem.setDetail(s"   dataObject $item")
+    completionItem.setInsertText(item)
     completionItem.setKind(CompletionItemKind.Snippet)
     completionItem
 

src/main/scala/io/smartdatalake/context/SDLBContext.scala

@@ -1,12 +1,14 @@
 package io.smartdatalake.context
 
-import com.typesafe.config.{Config, ConfigValue}
+import com.typesafe.config.{Config, ConfigList, ConfigObject, ConfigValue}
 import io.smartdatalake.context.TextContext
 import io.smartdatalake.context.TextContext.EMPTY_TEXT_CONTEXT
 import io.smartdatalake.context.hocon.HoconParser
 import io.smartdatalake.utils.MultiLineTransformer
 
-case class SDLBContext private(textContext: TextContext, parentPath: String, parentWord: String, word: String, oIndex: Option[Int]) {
+import scala.annotation.tailrec
+
+case class SDLBContext private(textContext: TextContext, parentPath: List[String], word: String, isInList: Boolean) {
 
   def withText(newText: String): SDLBContext = copy(textContext = textContext.update(newText))
 
@@ -15,22 +17,33 @@ case class SDLBContext private(textContext: TextContext, parentPath: String, par
     if originalLine <= 0 || originalLine > originalText.count(_ == '\n') + 1 || originalCol < 0 then this else
       val (newLine, newCol) = MultiLineTransformer.computeCorrectedPosition(originalText, originalLine, originalCol)
       val word = HoconParser.retrieveWordAtPosition(configText, newLine, newCol)
-      val (parentLine, parentWord) = HoconParser.retrieveDirectParent(configText, newLine, newCol)
-      val path = HoconParser.retrievePath(config, parentLine)
+      val (parentLine, _) = HoconParser.retrieveDirectParent(configText, newLine, newCol)
+      val (parentPathInitialList, isParentListKind) = HoconParser.retrievePathList(config, parentLine)
       val oIndex = HoconParser.findIndexIfInList(configText, newLine, newCol)
-      copy(parentPath = path, parentWord = parentWord, word = word, oIndex = oIndex)
-
-
-  //TODO keep that method?
-  def getParentContext: Option[ConfigValue] = if parentPath.isBlank then None else Some(textContext.config.getValue(parentPath))
-
+      val parentPath = oIndex match
+        case Some(index) => if isParentListKind then parentPathInitialList :+ index.toString else parentPathInitialList
+        case None => parentPathInitialList
+      val isInList = HoconParser.isInList(configText, newLine, newCol)
+      copy(parentPath = parentPath, word = word, isInList = isInList)
+
+
+  def getParentContext: Option[ConfigValue] =
+    @tailrec
+    def findParentContext(currentConfig: ConfigValue, remainingPath: List[String]): Option[ConfigValue] = remainingPath match
+      case Nil => Some(currentConfig)
+      case path::newRemainingPath => currentConfig match
+        case asConfigObject: ConfigObject => findParentContext(asConfigObject.get(path), newRemainingPath)
+        case asConfigList: ConfigList => findParentContext(asConfigList.get(path.toInt), newRemainingPath)
+        case _ => None
+
+    parentPath.headOption.flatMap(head => findParentContext(textContext.rootConfig.getValue(head), parentPath.tail))
 
 }
 
 object SDLBContext {
-  val EMPTY_CONTEXT: SDLBContext = SDLBContext(EMPTY_TEXT_CONTEXT, "", "", "", None)
+  val EMPTY_CONTEXT: SDLBContext = SDLBContext(EMPTY_TEXT_CONTEXT, List(), "", false)
 
-  def fromText(originalText: String): SDLBContext = SDLBContext(TextContext.create(originalText), "", "", "", None)
+  def fromText(originalText: String): SDLBContext = SDLBContext(TextContext.create(originalText), List(), "", false)
 
 }
 

src/main/scala/io/smartdatalake/context/TextContext.scala

@@ -5,7 +5,7 @@ import io.smartdatalake.context.TextContext.EMPTY_TEXT_CONTEXT
 import io.smartdatalake.context.hocon.HoconParser
 import io.smartdatalake.utils.MultiLineTransformer
 
-case class TextContext private (originalText: String, configText: String, config: Config) {
+case class TextContext private (originalText: String, configText: String, rootConfig: Config) {
 
   def update(newText: String): TextContext = this match
     case EMPTY_TEXT_CONTEXT => TextContext.create(newText)

src/main/scala/io/smartdatalake/context/hocon/HoconParser.scala

@@ -30,33 +30,38 @@ private[context] object HoconParser:
 
 
   /**
-   * Find the path corresponding to the line number
+   * Find the path corresponding to the line number.
+   * This version assumes last parent cannot be an index.
+   *
+   * Notice that this is a replacement from the old version "retrievePath" which provided a path like "a.b.c".
+   * Please see first commits to find back this version if necessary.
+   * Note for dev: Returning if last element of the path is a kind of list here is a bit hacky and ugly, but it avoids traversing the thing twice.
    * @param config a config representing the HOCON file
-   * @param line line number
-   * @return path in format "a.b.c"
+   * @param line   line number
+   * @return path in list format, and true if direct parent is of a list kind
    */
-  def retrievePath(config: Config, line: Int): String =
-    def matchTypeValueAndSearchRecursive(key: String, configValue: ConfigValue, currentPath: String): Option[String] = {
+  def retrievePathList(config: Config, line: Int): (List[String], Boolean) =
+    def matchTypeValueAndSearchRecursive(key: String, configValue: ConfigValue, currentPath: List[String]): Option[(List[String], Boolean)] = {
       configValue match
-        case configList: ConfigList => configList.toScala.zipWithIndex.flatMap{ (config, index) => matchTypeValueAndSearchRecursive(index.toString, config, currentPath + "." + key)}.headOption
-        case configObject: ConfigObject => searchPath(configObject, currentPath + "." + key)
+        case configList: ConfigList => configList.toScala.zipWithIndex.flatMap { (config, index) => matchTypeValueAndSearchRecursive(index.toString, config, key::currentPath) }.headOption
+        case configObject: ConfigObject => searchPath(configObject, key::currentPath)
         case _ => None
     }
 
-    def searchPath(currentConfig: ConfigObject, currentPath: String): Option[String] =
+    def searchPath(currentConfig: ConfigObject, currentPath: List[String]): Option[(List[String], Boolean)] =
       import scala.jdk.CollectionConverters._
 
       val entrySet = currentConfig.entrySet().asScala
 
       entrySet.find(_.getValue.origin().lineNumber() == line) match
         case Some(entry) =>
-          Some(currentPath + "." + entry.getKey)
+          Some((entry.getKey::currentPath, entry.getValue.isInstanceOf[ConfigList]))
         case None =>
           entrySet.flatMap { entry => matchTypeValueAndSearchRecursive(entry.getKey, entry.getValue, currentPath)
           }.headOption
 
-    searchPath(config.root(), "").getOrElse("").stripPrefix(".")
-
+    val (l, b) = searchPath(config.root(), List.empty[String]).getOrElse((List.empty[String], false))
+    (l.reverse, b)
 
   /**
    * Retrieve the direct parent of the current caret position.
@@ -69,31 +74,51 @@ private[context] object HoconParser:
    */
   def retrieveDirectParent(text: String, line: Int, column: Int): (Int, String) =
 
-    @tailrec
     /*
     Note that retrieveHelper has not the exact same logic than retrieveDirectParent.
     */
+    @tailrec
     def retrieveHelper(line: Int, depth: Int): (Int, String) = {
       if line <= 0 then
         (0, "")
       else
-        val textLine = text.split(Token.NEW_LINE)(line-1).filterNot(c => c.isWhitespace || c == Token.START_LIST || c == Token.END_LIST).takeWhile(_ != Token.COMMENT).mkString
-        val newDepth = depth - textLine.count(_ == Token.START_OBJECT) + textLine.count(_ == Token.END_OBJECT) - textLine.count(_ == Token.END_LIST) + textLine.count(_ == Token.START_LIST)
-        if textLine.contains(Token.END_OBJECT) then retrieveHelper(line-1, newDepth) else
+        val textLine = text.split(Token.NEW_LINE)(line-1).filterNot(c => c.isWhitespace).takeWhile(_ != Token.COMMENT).mkString
+        val newDepth = depth - textLine.count(_ == Token.START_OBJECT) + textLine.count(_ == Token.END_OBJECT) + textLine.count(_ == Token.END_LIST) - textLine.count(_ == Token.START_LIST)
+        if textLine.contains(Token.END_OBJECT) || textLine.contains(Token.END_LIST) then retrieveHelper(line-1, newDepth) else
           textLine.split(Token.KEY_VAL_SPLIT_REGEX) match
-            case Array(singleBlock) => if singleBlock.isBlank then retrieveHelper(line-1, newDepth) else if depth == 0 then (line, singleBlock) else retrieveHelper(line-1, newDepth)
+            case Array(singleBlock) => if singleBlock.isBlank then retrieveHelper(line-1, newDepth) else if newDepth < 0 then (line, singleBlock) else retrieveHelper(line-1, newDepth)
             case _ => retrieveHelper(line-1, newDepth)
     }
 
     val textLine = text.split(Token.NEW_LINE)(line-1).takeWhile(_ != Token.COMMENT).mkString
     val col = math.min(textLine.length, column)
-    if textLine.count(_ == Token.END_OBJECT) > textLine.count(_ == Token.START_OBJECT) then retrieveHelper(line-1, if col > textLine.indexOf(Token.END_OBJECT) then 1 else 0) else
+    if textLine.count(_ == Token.END_OBJECT) + textLine.count(_ == Token.END_LIST) > textLine.count(_ == Token.START_OBJECT) + textLine.count(_ == Token.START_LIST) then
+      val depthForObject = if textLine.indexOf(Token.END_OBJECT) != -1 && col > textLine.indexOf(Token.END_OBJECT) then 1 else 0
+      val depthForList   = if textLine.indexOf(Token.END_LIST) != -1 && col > textLine.indexOf(Token.END_LIST) then 1 else 0
+      retrieveHelper(line-1, depthForObject + depthForList)
+    else
       val keyValSplit = textLine.split(Token.KEY_VAL_SPLIT_REGEX)
       if col > keyValSplit(0).length then
         val keyName = keyValSplit(0).trim
-        (if keyName.isBlank then 0 else line, keyName)
+        @tailrec
+        def findValidDirectParentName(line: Int): Option[(Int, String)] = if line <= 0 then None else //TODO test
+          val textLine = text.split(Token.NEW_LINE)(line-1).takeWhile(_ != Token.COMMENT).mkString
+          if textLine.isBlank then findValidDirectParentName(line-1) else
+            val words = textLine.filterNot(c => c == Token.START_LIST || c == Token.END_LIST || c == Token.START_OBJECT || c == Token.END_OBJECT || c == '=' || c == '"').split(" ")
+            words match
+              case Array(singleBlock) => Some(line, singleBlock)
+              case _ => None
+        end findValidDirectParentName
+
+        if keyName.isBlank then
+          val oParentName = findValidDirectParentName(line-1)
+          oParentName.getOrElse(retrieveHelper(line-1, 0))
+        else
+        (line, keyName)
       else
         retrieveHelper(line-1, 0)
+  end retrieveDirectParent
+
 
 
   def retrieveWordAtPosition(text: String, line: Int, col: Int): String =
@@ -148,6 +173,10 @@ private[context] object HoconParser:
 
   private[hocon] def findObjectAreaFrom(text: String, position: Int): Option[(Int, Int)] = findAreaFrom(text, position, Token.START_OBJECT, Token.END_OBJECT)
   private[hocon] def findListAreaFrom(text: String, position: Int): Option[(Int, Int)] = findAreaFrom(text, position, Token.START_LIST, Token.END_LIST)
+
+  def isInList(text: String, line: Int, column: Int): Boolean =
+    val absolutePosition = lineColToAbsolutePosition(text, line, column)
+    findListAreaFrom(text, absolutePosition).isDefined
 
   private def findAreaFrom(text: String, position: Int, startToken: Char, endToken: Char): Option[(Int, Int)] =
     @tailrec

src/main/scala/io/smartdatalake/context/hocon/HoconTokens.scala

@@ -1,7 +1,7 @@
 package io.smartdatalake.context.hocon
 
 object HoconTokens {
-  val KEY_VAL_SPLIT_REGEX = "[{=]"
+  val KEY_VAL_SPLIT_REGEX = "[\\[{=]"
   val START_OBJECT = '{'
   val END_OBJECT = '}'
   val START_LIST = '['

src/main/scala/io/smartdatalake/conversions/ScalaJavaConverter.scala

@@ -9,6 +9,7 @@ import scala.concurrent.Future
 import scala.jdk.FutureConverters.*
 import scala.jdk.CollectionConverters.*
 import java.util.List as JList
+import java.util.Set as JSet
 
 trait ScalaJavaConverter {
 
@@ -18,6 +19,8 @@ trait ScalaJavaConverter {
 
   extension [T] (l: JList[T]) def toScala: List[T] = l.asScala.toList
 
+  extension [T] (s: JSet[T]) def toScala: Set[T] = s.asScala.toSet
+
   extension [L, R] (either: Either[L, R]) def toJava: messages.Either[L, R] = either match
     case Left(leftValue) => messages.Either.forLeft(leftValue)
     case Right(rightValue) => messages.Either.forRight(rightValue) 

src/main/scala/io/smartdatalake/hover/SDLBHoverEngineImpl.scala

@@ -13,13 +13,13 @@ class SDLBHoverEngineImpl(private val schemaReader: SchemaReader) extends SDLBHo
     markupContent.setValue(retrieveSchemaDescription(context))
     new Hover(markupContent)
 
-  private def retrieveSchemaDescription(context: SDLBContext): String =
-    context.parentPath match
-      case path if path.startsWith("actions") && path.count(_ == '.') == 1 =>
-        val tActionType: Try[String] = Try(context.textContext.config.getString(context.parentPath + ".type"))
-        tActionType match
-          case Success(actionType) => schemaReader.retrieveActionPropertyDescription(actionType, context.word)
-          case Failure(_) => ""
-
-      case _ => ""
+  private def retrieveSchemaDescription(context: SDLBContext): String = context.parentPath.mkString(".")
+//    context.parentPath match
+//      case path if path.startsWith("actions") && path.count(_ == '.') == 1 =>
+//        val tActionType: Try[String] = Failure(new IllegalStateException()) // Try(context.textContext.rootConfig.getString(context.parentPath + ".type"))
+//        tActionType match
+//          case Success(actionType) => schemaReader.retrieveActionPropertyDescription(actionType, context.word)
+//          case Failure(_) => ""
+//
+//      case _ => ""
 

src/main/scala/io/smartdatalake/schema/ItemType.scala

@@ -1,5 +1,7 @@
 package io.smartdatalake.schema
 
+import org.slf4j.LoggerFactory
+
 enum ItemType(val name: String, val defaultValue: String) {
   case STRING extends ItemType("string", "\"???\"")
   case BOOLEAN extends ItemType("boolean", "true")
@@ -14,9 +16,14 @@ enum ItemType(val name: String, val defaultValue: String) {
 }
 
 object ItemType:
+  private val logger = LoggerFactory.getLogger(ItemType.getClass)
   def fromName(name: String): ItemType = name match
     case "string" => ItemType.STRING
     case "boolean" => ItemType.BOOLEAN
     case "integer" => ItemType.INTEGER
     case "object" => ItemType.OBJECT
-    case "array" => ItemType.ARRAY
+    case "array" => ItemType.ARRAY
+    case _ => 
+      logger.warn("Attempt to translate unknown type: {}", name)
+      ItemType.STRING
+

src/main/scala/io/smartdatalake/schema/SchemaCollections.scala

@@ -0,0 +1,9 @@
+package io.smartdatalake.schema
+
+import scala.collection.Iterable
+
+object SchemaCollections {
+  case class AttributeCollection(attributes: Iterable[SchemaItem])
+  case class TemplateCollection(templates: Iterable[(String, Iterable[SchemaItem])])
+
+}