Refactor context suggestions in ContextAdvisor + hovering stability i…

…mprovement + handling multiple opened files in the same time

src/main/scala/io/smartdatalake/completion/SDLBCompletionEngineImpl.scala

@@ -2,36 +2,25 @@ package io.smartdatalake.completion
 
 import com.typesafe.config.{Config, ConfigList, ConfigObject, ConfigValue}
 import io.smartdatalake.completion.SDLBCompletionEngine
-import io.smartdatalake.context.{SDLBContext, TextContext}
+import io.smartdatalake.context.{ContextAdvisor, ContextSuggestion, SDLBContext, TextContext}
 import io.smartdatalake.schema.SchemaCollections.{AttributeCollection, TemplateCollection}
 import io.smartdatalake.schema.{ItemType, SchemaItem, SchemaReader, SchemaReaderImpl, TemplateType}
 import io.smartdatalake.conversions.ScalaJavaConverterAPI.*
 import org.eclipse.lsp4j.{CompletionItem, CompletionItemKind}
 
 import scala.util.{Failure, Success, Try}
 
-class SDLBCompletionEngineImpl(private val schemaReader: SchemaReader) extends SDLBCompletionEngine {
+class SDLBCompletionEngineImpl(private val schemaReader: SchemaReader, private val contextAdvisor: ContextAdvisor) extends SDLBCompletionEngine {
 
   override def generateCompletionItems(context: SDLBContext): List[CompletionItem] =
     val itemSuggestionsFromSchema = schemaReader.retrieveAttributeOrTemplateCollection(context) match
       case AttributeCollection(attributes) => generateAttributeSuggestions(attributes, context.getParentContext)
       case TemplateCollection(templates, templateType) => generateTemplateSuggestions(templates, templateType)
 
-    val itemSuggestionsFromConfig = generateItemSuggestionsFromConfig(context)
-    val allItems = itemSuggestionsFromConfig ++ itemSuggestionsFromSchema //TODO better split schema and config suggestions
-    if allItems.isEmpty then typeList else allItems
-
-  private def generateItemSuggestionsFromConfig(context: SDLBContext): List[CompletionItem] = context.parentPath.lastOption match
-    case Some(value) => value match
-      case "inputId" | "outputId" | "inputIds" | "outputIds" => retrieveDataObjectIds(context) //TODO regex?
-      case _ => List.empty[CompletionItem]
-    case None => List.empty[CompletionItem]
-
-  private def retrieveDataObjectIds(context: SDLBContext): List[CompletionItem] = //TODO test. TODO return List[String] instead?
-    context.textContext.rootConfig.getValue("dataObjects") match
-      case asConfigObject: ConfigObject => asConfigObject.unwrapped().keySet().toScala.toList.map(createCompletionItem)
-
-      case _ => List.empty[CompletionItem]
+    val itemSuggestionsFromConfig = contextAdvisor.generateSuggestions(context).map(createCompletionItem)
+    val allItems = itemSuggestionsFromConfig ++ itemSuggestionsFromSchema
+    if allItems.isEmpty then typeList else allItems //TODO wrong
+
   private def generateAttributeSuggestions(attributes: Iterable[SchemaItem], parentContext: Option[ConfigValue]): List[CompletionItem] =
     val items = parentContext match
       case Some(config: ConfigObject) => attributes.filter(item => Option(config.get(item.name)).isEmpty)
@@ -66,11 +55,11 @@ class SDLBCompletionEngineImpl(private val schemaReader: SchemaReader) extends S
     completionItem.setKind(CompletionItemKind.Snippet)
     completionItem
 
-  private def createCompletionItem(item: String): CompletionItem =
+  private def createCompletionItem(item: ContextSuggestion): CompletionItem =
     val completionItem = new CompletionItem()
-    completionItem.setLabel(item)
-    completionItem.setDetail(s"   dataObject $item")
-    completionItem.setInsertText(item)
+    completionItem.setLabel(item.value)
+    completionItem.setDetail(s"   ${item.label}")
+    completionItem.setInsertText(item.value)
     completionItem.setKind(CompletionItemKind.Snippet)
     completionItem
 

src/main/scala/io/smartdatalake/context/ContextAdvisor.scala

@@ -0,0 +1,4 @@
+package io.smartdatalake.context
+
+trait ContextAdvisor:
+  def generateSuggestions(context: SDLBContext): List[ContextSuggestion]

src/main/scala/io/smartdatalake/context/ContextAdvisorImpl.scala

@@ -0,0 +1,26 @@
+package io.smartdatalake.context
+
+import com.typesafe.config.ConfigObject
+import org.eclipse.lsp4j.CompletionItem
+import java.util.Map as JMap
+
+import scala.collection.immutable.List
+
+import io.smartdatalake.conversions.ScalaJavaConverterAPI.*
+
+
+class ContextAdvisorImpl extends ContextAdvisor:
+  override def generateSuggestions(context: SDLBContext): List[ContextSuggestion] = context.parentPath.lastOption match
+    case Some(value) => value match
+      case "inputId" | "outputId" | "inputIds" | "outputIds" => retrieveDataObjectIds(context)
+      case _ => List.empty[ContextSuggestion]
+    case None => List.empty[ContextSuggestion]
+
+  private def retrieveDataObjectIds(context: SDLBContext): List[ContextSuggestion] =
+    Option(context.textContext.rootConfig.root().get("dataObjects")) match
+      case Some(asConfigObject: ConfigObject) => asConfigObject.unwrapped().toScala.map { (k, v) => v match
+        case jMap: JMap[String, Object] => ContextSuggestion(k, Option(jMap.get("type")).map(_.toString).getOrElse(""))
+        case _ => ContextSuggestion(k, "")
+      }.toList
+      case _ => List.empty[ContextSuggestion]
+

src/main/scala/io/smartdatalake/context/ContextSuggestion.scala

@@ -0,0 +1,3 @@
+package io.smartdatalake.context
+
+case class ContextSuggestion(value: String, label: String)

src/main/scala/io/smartdatalake/context/TextContext.scala

@@ -16,6 +16,8 @@ case class TextContext private (originalText: String, configText: String, rootCo
     val newConfig = HoconParser.parse(newConfigText).getOrElse(HoconParser.EMPTY_CONFIG)
     if newConfig == HoconParser.EMPTY_CONFIG then this else TextContext(newText, newConfigText, newConfig)
 
+  override def toString: String = s"TextContext(originalText=${originalText.take(50)}, configText=${configText.take(50)}, rootConfig=${rootConfig.toString.take(50)})"
+
 
 }
 

src/main/scala/io/smartdatalake/conversions/ScalaJavaConverter.scala

@@ -10,6 +10,7 @@ import scala.jdk.FutureConverters.*
 import scala.jdk.CollectionConverters.*
 import java.util.List as JList
 import java.util.Set as JSet
+import java.util.Map as JMap
 
 trait ScalaJavaConverter {
 
@@ -21,6 +22,8 @@ trait ScalaJavaConverter {
 
   extension [T] (s: JSet[T]) def toScala: Set[T] = s.asScala.toSet
 
+  extension[T, U] (m: JMap[T, U]) def toScala: Map[T, U] = m.asScala.toMap
+
   extension [L, R] (either: Either[L, R]) def toJava: messages.Either[L, R] = either match
     case Left(leftValue) => messages.Either.forLeft(leftValue)
     case Right(rightValue) => messages.Either.forRight(rightValue) 

src/main/scala/io/smartdatalake/languageserver/SmartDataLakeTextDocumentService.scala

@@ -8,6 +8,7 @@ import io.smartdatalake.conversions.ScalaJavaConverterAPI.*
 import org.eclipse.lsp4j.jsonrpc.messages
 import org.eclipse.lsp4j.services.TextDocumentService
 import org.eclipse.lsp4j.{CodeAction, CodeActionParams, CodeLens, CodeLensParams, Command, CompletionItem, CompletionItemKind, CompletionList, CompletionParams, DefinitionParams, DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams, DocumentFormattingParams, DocumentHighlight, DocumentHighlightParams, DocumentOnTypeFormattingParams, DocumentRangeFormattingParams, DocumentSymbol, DocumentSymbolParams, Hover, HoverParams, InsertReplaceEdit, Location, LocationLink, MarkupContent, MarkupKind, Position, Range, ReferenceParams, RenameParams, SignatureHelp, SignatureHelpParams, SymbolInformation, TextDocumentPositionParams, TextEdit, WorkspaceEdit}
+import org.slf4j.LoggerFactory
 
 import java.util
 import java.util.concurrent.CompletableFuture
@@ -17,11 +18,12 @@ import scala.util.Using
 
 class SmartDataLakeTextDocumentService(private val completionEngine: SDLBCompletionEngine, private val hoverEngine: SDLBHoverEngine)(using ExecutionContext) extends TextDocumentService {
 
-  private var context: SDLBContext = SDLBContext.EMPTY_CONTEXT
+  private var uriToContextMap: Map[String, SDLBContext] = Map("" -> SDLBContext.EMPTY_CONTEXT)
 
   override def completion(params: CompletionParams): CompletableFuture[messages.Either[util.List[CompletionItem], CompletionList]] = {
 
     Future {
+      val context = uriToContextMap(params.getTextDocument.getUri)
       val caretContext = context.withCaretPosition(params.getPosition.getLine+1, params.getPosition.getCharacter)
       val completionItems: util.List[CompletionItem] = completionEngine.generateCompletionItems(caretContext).toJava
       Left(completionItems).toJava
@@ -30,28 +32,32 @@ class SmartDataLakeTextDocumentService(private val completionEngine: SDLBComplet
   }
 
   override def didOpen(didOpenTextDocumentParams: DidOpenTextDocumentParams): Unit =
-    context = SDLBContext.fromText(didOpenTextDocumentParams.getTextDocument.getText)
+    uriToContextMap += (didOpenTextDocumentParams.getTextDocument.getUri, SDLBContext.fromText(didOpenTextDocumentParams.getTextDocument.getText))
 
   override def didChange(didChangeTextDocumentParams: DidChangeTextDocumentParams): Unit =
     val contentChanges = didChangeTextDocumentParams.getContentChanges
+    val context = uriToContextMap(didChangeTextDocumentParams.getTextDocument.getUri)
     val newContext =
       if contentChanges != null && contentChanges.size() > 0 then
         // Update the stored document content with the new content. Assuming Full sync technique
         context.withText(contentChanges.get(0).getText)
       else
         context
-    context = newContext
+    uriToContextMap += (didChangeTextDocumentParams.getTextDocument.getUri, newContext)
 
 
-  override def didClose(didCloseTextDocumentParams: DidCloseTextDocumentParams): Unit = ???
+  override def didClose(didCloseTextDocumentParams: DidCloseTextDocumentParams): Unit = uriToContextMap -= didCloseTextDocumentParams.getTextDocument.getUri
 
   override def didSave(didSaveTextDocumentParams: DidSaveTextDocumentParams): Unit = ???
 
   override def resolveCompletionItem(completionItem: CompletionItem): CompletableFuture[CompletionItem] = ???
 
   override def hover(params: HoverParams): CompletableFuture[Hover] = {
     Future {
+      val context = uriToContextMap(params.getTextDocument.getUri)
       val hoverContext = context.withCaretPosition(params.getPosition.getLine + 1, params.getPosition.getCharacter)
+      val logger = LoggerFactory.getLogger(getClass)
+      logger.trace("Attempt to hover with hoverContext={}", hoverContext)
       hoverEngine.generateHoveringInformation(hoverContext)
     }.toJava
   }

src/main/scala/io/smartdatalake/modules/AppModule.scala

@@ -1,6 +1,7 @@
 package io.smartdatalake.modules
 
 import io.smartdatalake.completion.{SDLBCompletionEngine, SDLBCompletionEngineImpl}
+import io.smartdatalake.context.{ContextAdvisor, ContextAdvisorImpl}
 import io.smartdatalake.hover.{SDLBHoverEngine, SDLBHoverEngineImpl}
 import io.smartdatalake.languageserver.{SmartDataLakeLanguageServer, SmartDataLakeTextDocumentService, SmartDataLakeWorkspaceService}
 import io.smartdatalake.schema.{SchemaReader, SchemaReaderImpl}
@@ -11,7 +12,8 @@ import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService}
 
 trait AppModule {
   lazy val schemaReader: SchemaReader = new SchemaReaderImpl("sdl-schema/sdl-schema-2.5.0.json")
-  lazy val completionEngine: SDLBCompletionEngine = new SDLBCompletionEngineImpl(schemaReader)
+  lazy val contextAdvisor: ContextAdvisor = new ContextAdvisorImpl
+  lazy val completionEngine: SDLBCompletionEngine = new SDLBCompletionEngineImpl(schemaReader, contextAdvisor)
   lazy val hoverEngine: SDLBHoverEngine = new SDLBHoverEngineImpl(schemaReader)
   lazy val executorService: ExecutorService = Executors.newCachedThreadPool()
   lazy val executionContext: ExecutionContext & ExecutorService = ExecutionContext.fromExecutorService(executorService)

src/main/scala/io/smartdatalake/schema/SchemaReaderImpl.scala

@@ -38,22 +38,20 @@ class SchemaReaderImpl(val schemaPath: String) extends SchemaReader {
     word.filterNot(Set('{', '}', '[', ']', '"', '(', ')', '#', '/', '\\', '.').contains(_)).isBlank
 
   private[schema] def retrieveSchemaContext(context: SDLBContext, withWordInPath: Boolean): Option[SchemaContext] =
-    val rootConfig = context.textContext.rootConfig
     val path = if withWordInPath then context.parentPath.appended(context.word) else context.parentPath
-    path match
-      case Nil => Some(createGlobalSchemaContext)
-      case globalObject::remainingPath =>
-        val schemaContext = createGlobalSchemaContext.updateByName(globalObject)
-        val rootConfigValue = rootConfig.getValue(globalObject)
-        remainingPath.foldLeft((schemaContext, rootConfigValue)){(scCv, elementPath) =>
-          val (newConfigValue, oTypeObject) = moveInConfigAndRetrieveType(scCv._2, elementPath)
-          val newSchemaContext = oTypeObject match
-            case Some(objectType) =>
-              val tryUpdateByName = scCv._1.flatMap(_.updateByName(elementPath))
-              tryUpdateByName.orElse(scCv._1).flatMap(_.updateByType(objectType))
-            case None => scCv._1.flatMap(_.updateByName(elementPath))
-          (newSchemaContext, newConfigValue)
-        }._1
+    val oInitialSchemaContext: Option[SchemaContext] = Some(createGlobalSchemaContext)
+    val rootConfigValue: ConfigValue = context.textContext.rootConfig.root()
+    logger.debug("path = {}", path)
+    path.foldLeft((oInitialSchemaContext, rootConfigValue)){(scCv, elementPath) =>
+      val (newConfigValue, oTypeObject) = moveInConfigAndRetrieveType(scCv._2, elementPath)
+      if (newConfigValue == null) {logger.error("Error, newConfig is null with pathElement={} and fullPath={}", elementPath, path)}
+      val newSchemaContext = oTypeObject match
+        case Some(objectType) =>
+          val tryUpdateByName = scCv._1.flatMap(_.updateByName(elementPath))
+          tryUpdateByName.orElse(scCv._1).flatMap(_.updateByType(objectType))
+        case None => scCv._1.flatMap(_.updateByName(elementPath))
+      (newSchemaContext, newConfigValue)
+    }._1
   end retrieveSchemaContext
 
   private[schema] def moveInConfigAndRetrieveType(config: ConfigValue, path: String): (ConfigValue, Option[String]) =
@@ -68,7 +66,6 @@ class SchemaReaderImpl(val schemaPath: String) extends SchemaReader {
         config
 
     val objectType = retrieveType(newConfig)
-    if (newConfig == null) {logger.error("Error, newConfig is null with path={}, config={}", path, config)}
     (newConfig, objectType)
 
   private def retrieveType(config: ConfigValue): Option[String] = config match

src/test/scala/io/smartdatalake/context/ContextAdvisorSpec.scala

@@ -0,0 +1,25 @@
+package io.smartdatalake.context
+
+import com.typesafe.config.{Config, ConfigRenderOptions, ConfigUtil}
+import io.smartdatalake.UnitSpec
+import io.smartdatalake.context.hocon.HoconParser
+import io.smartdatalake.utils.MultiLineTransformer as MLT
+
+import scala.io.Source
+import scala.util.Using
+
+class ContextAdvisorSpec extends UnitSpec {
+
+  private val context: SDLBContext = SDLBContext.fromText(loadFile("fixture/hocon/airport-example.conf"))
+
+
+  "Context Advisor" should "generate correctly dataObject suggestions when on inputId attribute" in {
+    val inputIdContext = context.withCaretPosition(62, 14)
+    val suggestions = contextAdvisor.generateSuggestions(inputIdContext)
+    suggestions should have size 7
+    suggestions should contain (ContextSuggestion("ext-airports", "WebserviceFileDataObject"))
+    suggestions should contain (ContextSuggestion("btl-distances", "CsvFileDataObject"))
+
+  }
+
+}

src/test/scala/io/smartdatalake/context/SDLBContextSpec.scala

@@ -72,23 +72,7 @@ class SDLBContextSpec extends UnitSpec {
     val line24EdgeInsideAgain = SDLBContext.fromText(withListText).withCaretPosition(24, 7)
     line24EdgeInsideAgain.parentPath shouldBe List("actions", "join-departures-airports", "transformers", "1")
     line24EdgeInsideAgain.word shouldBe "{"
-
-    //TODO:
-    // 3. Adapt SDLBCompletionEngine and HoverEngine
-    // 4. Adapt all necessary tests => especially SchemaContextSpec and SDLBCompletionEngineSpec and SDLBHoverEngineSpec and maybe SchemaReaderSpec
-
-    // KNOWN PROBLEMS
-    // 1. executionMode has its oneOf not generating templates
-    // >>>>>>>>>2. executionMode with a provided type does not generate properties <<<<<<< SOLVED
-    // >>>>>>>>>3. executionMode is shown as being a String which is wrong <<<<<<<<<< SOLVED
-    // >>>>4. Templates within element of lists should be anonymous... See how to generate them anyway then <<<<< SOLVED
-    // 5. Tabulation format: Indentation is not correctly generated
-    // >>>>6. Missing intelligence between DataObjects and Actions <<< SOLVED
-    // 7. Redo Hovering
-    // >>>>>>>8. Items are not removed if already existing <<<<<<< SOLVED
-    // >>>>>>>9. If type is not provided it should be suggested. <<<<<< SOLVED
-    // 10. Values of type should be suggested.
-    // 11. Suggest 4 basic properties when in root level
+
   }
 
   it should "create a context correctly when in a list but not in an element directly" in {

src/test/scala/io/smartdatalake/languageserver/SmartDataLakeTextDocumentServiceSpec.scala

@@ -2,14 +2,17 @@ package io.smartdatalake.languageserver
 
 import io.smartdatalake.UnitSpec
 import io.smartdatalake.languageserver.SmartDataLakeTextDocumentService
-import org.eclipse.lsp4j.{CompletionParams, DidOpenTextDocumentParams, HoverParams, Position, TextDocumentItem}
+import org.eclipse.lsp4j.{CompletionParams, DidOpenTextDocumentParams, HoverParams, Position, TextDocumentIdentifier, TextDocumentItem}
 
 class SmartDataLakeTextDocumentServiceSpec extends UnitSpec {
 
   def params: CompletionParams =
     val p = new CompletionParams()
     // Careful, Position of LSP4J is 0-based
     p.setPosition(new Position(16, 0))
+    val textDocumentIdentifier = new TextDocumentIdentifier()
+    textDocumentIdentifier.setUri("example.conf")
+    p.setTextDocument(textDocumentIdentifier)
     p
 
   "SDL text document service" should "suggest at least one autocompletion item" in {
@@ -24,13 +27,17 @@ class SmartDataLakeTextDocumentServiceSpec extends UnitSpec {
     val params = new HoverParams()
     // Careful, Position of LSP4J is 0-based
     params.setPosition(new Position(5, 4))
+    val textDocumentIdentifier = new TextDocumentIdentifier()
+    textDocumentIdentifier.setUri("example.conf")
+    params.setTextDocument(textDocumentIdentifier)
     val hoverInformation = textDocumentService.hover(params)
     assert(!hoverInformation.get().getContents.getRight.getValue.isBlank)
   }
   private def notifyOpenFile(): Unit = {
     val didOpenTextDocumentParams: DidOpenTextDocumentParams = new DidOpenTextDocumentParams()
     val textDocumentItem: TextDocumentItem = new TextDocumentItem()
     textDocumentItem.setText(loadFile("fixture/hocon/with-multi-lines-example.conf"))
+    textDocumentItem.setUri("example.conf")
     didOpenTextDocumentParams.setTextDocument(textDocumentItem)
     textDocumentService.didOpen(didOpenTextDocumentParams)
   }

src/test/scala/io/smartdatalake/modules/TestModule.scala

@@ -5,7 +5,7 @@ import io.smartdatalake.schema.{SchemaReader, SchemaReaderImpl}
 
 trait TestModule extends AppModule {
   override lazy val schemaReader: SchemaReaderImpl = new SchemaReaderImpl("fixture/sdl-schema/sdl-schema-2.5.0.json")
-  override lazy val completionEngine: SDLBCompletionEngineImpl = new SDLBCompletionEngineImpl(schemaReader)
+  override lazy val completionEngine: SDLBCompletionEngineImpl = new SDLBCompletionEngineImpl(schemaReader, contextAdvisor)
   override lazy val languageServer: SmartDataLakeLanguageServer = new SmartDataLakeLanguageServer(textDocumentService, workspaceService)(using executionContext)
 
 }