Native Language Server integration with PM

build.sbt

@@ -2170,11 +2170,14 @@ lazy val `engine-common` = project
     libraryDependencies ++= Seq(
       "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided"
     ),
+    libraryDependencies ++= GraalVM.modules.map(
+      _.withConfigurations(Some(Runtime.name))
+    ),
     Compile / moduleDependencies ++= {
       Seq(
         "org.graalvm.polyglot" % "polyglot"  % graalMavenPackagesVersion,
         "org.slf4j"            % "slf4j-api" % slf4jVersion
-      )
+      ) ++ GraalVM.modules.map(_.withConfigurations(Some(Runtime.name)))
     },
     Compile / internalModuleDependencies := Seq(
       (`logging-utils` / Compile / exportedModule).value,
@@ -2280,16 +2283,21 @@ lazy val `language-server` = (project in file("engine/language-server"))
       "org.eclipse.jgit"    % "org.eclipse.jgit"        % jgitVersion,
       "org.apache.tika"     % "tika-core"               % tikaVersion               % Test
     ),
+    libraryDependencies ++= GraalVM.modules.map(
+      _.withConfigurations(Some(Runtime.name))
+    ),
     javaModuleName := "org.enso.language.server",
     Compile / moduleDependencies ++=
       Seq(
-        "org.graalvm.polyglot"   % "polyglot"         % graalMavenPackagesVersion,
-        "org.slf4j"              % "slf4j-api"        % slf4jVersion,
-        "commons-cli"            % "commons-cli"      % commonsCliVersion,
-        "commons-io"             % "commons-io"       % commonsIoVersion,
-        "com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion,
-        "org.eclipse.jgit"       % "org.eclipse.jgit" % jgitVersion
-      ),
+        "org.graalvm.polyglot"   % "polyglot"                % graalMavenPackagesVersion,
+        "org.graalvm.truffle"    % "truffle-api"             % graalMavenPackagesVersion,
+        "org.slf4j"              % "slf4j-api"               % slf4jVersion,
+        "commons-cli"            % "commons-cli"             % commonsCliVersion,
+        "commons-io"             % "commons-io"              % commonsIoVersion,
+        "com.google.flatbuffers" % "flatbuffers-java"        % flatbuffersVersion,
+        "org.eclipse.jgit"       % "org.eclipse.jgit"        % jgitVersion,
+        "org.netbeans.api"       % "org-openide-util-lookup" % netbeansApiVersion
+      ) ++ GraalVM.modules.map(_.withConfigurations(Some(Runtime.name))),
     Compile / internalModuleDependencies := Seq(
       (`akka-wrapper` / Compile / exportedModule).value,
       (`zio-wrapper` / Compile / exportedModule).value,
@@ -3614,13 +3622,17 @@ lazy val `engine-runner` = project
       val epbLang =
         (`runtime-language-epb` / Compile / fullClasspath).value
           .map(_.data.getAbsolutePath)
+      val langServer =
+        (`language-server` / Compile / fullClasspath).value
+          .map(_.data.getAbsolutePath)
       val core = (
         runnerDeps ++
           runtimeDeps ++
           loggingDeps ++
           replDebugInstr ++
           runtimeServerInstr ++
           idExecInstr ++
+          langServer ++
           epbLang
       ).distinct
       val stdLibsJars =
@@ -3705,6 +3717,7 @@ lazy val `engine-runner` = project
               "-H:IncludeResources=.*Main.enso$",
               "-H:+AddAllCharsets",
               "-H:+IncludeAllLocales",
+              "-H:+UnlockExperimentalVMOptions",
               "-ea",
               // useful perf & debug switches:
               // "-g",
@@ -3721,6 +3734,7 @@ lazy val `engine-runner` = project
               "org.jline",
               "io.methvin.watchservice",
               "zio.internal",
+              "zio",
               "org.enso.runner",
               "sun.awt",
               "sun.java2d",
@@ -3732,7 +3746,8 @@ lazy val `engine-runner` = project
               "akka.http",
               "org.enso.base",
               "org.enso.image",
-              "org.enso.table"
+              "org.enso.table",
+              "org.eclipse.jgit"
             )
           )
       }

engine/common/src/main/java/org/enso/common/ContextFactory.java

@@ -9,6 +9,7 @@
 import org.enso.logger.JulHandler;
 import org.enso.logging.config.LoggerSetup;
 import org.graalvm.polyglot.Context;
+import org.graalvm.polyglot.Engine;
 import org.graalvm.polyglot.HostAccess;
 import org.graalvm.polyglot.io.MessageTransport;
 import org.slf4j.event.Level;
@@ -52,6 +53,7 @@ public final class ContextFactory {
   private String checkForWarnings;
   private int warningsLimit = 100;
   private java.util.Map<String, String> options = new HashMap<>();
+  private java.util.Map<String, String> engineOptions = null;
   private boolean enableDebugServer;
 
   private ContextFactory() {}
@@ -145,6 +147,11 @@ public ContextFactory options(Map<String, String> options) {
     return this;
   }
 
+  public ContextFactory engineOptions(Map<String, String> options) {
+    this.engineOptions = options;
+    return this;
+  }
+
   public ContextFactory checkForWarnings(String fqnOfMethod) {
     this.checkForWarnings = fqnOfMethod;
     return this;
@@ -189,9 +196,6 @@ public Context build() {
     if (enableDebugServer) {
       builder.option(DebugServerInfo.ENABLE_OPTION, "true");
     }
-    if (messageTransport != null) {
-      builder.serverTransport(messageTransport);
-    }
     builder.option(RuntimeOptions.LOG_LEVEL, logLevelName);
     var logHandler = JulHandler.get();
     var logLevels = LoggerSetup.get().getConfig().getLoggers();
@@ -228,6 +232,19 @@ public Context build() {
           .allowCreateThread(true);
     }
 
+    if (engineOptions != null) {
+      // In AOT mode one must not use a shared engine; the latter causes issues when initializing
+      // message transport - it is set to `null`.
+      var eng = Engine.newBuilder().allowExperimentalOptions(true).options(engineOptions);
+
+      if (messageTransport != null) {
+        eng.serverTransport(messageTransport);
+      }
+      builder.engine(eng.build());
+    } else if (messageTransport != null) {
+      builder.serverTransport(messageTransport);
+    }
+
     var ctx = builder.build();
     ContextInsightSetup.configureContext(ctx);
     return ctx;

engine/language-server/src/main/java/module-info.java

@@ -7,7 +7,7 @@
   requires commons.cli;
   requires flatbuffers.java;
   requires org.apache.commons.io;
-  requires org.graalvm.polyglot;
+  requires org.graalvm.truffle;
   requires org.eclipse.jgit;
   requires org.slf4j;
 
@@ -37,8 +37,8 @@
   requires org.enso.text.buffer;
   requires org.enso.task.progress.notifications;
   requires org.enso.ydoc.polyfill;
+  requires org.openide.util.lookup.RELEASE180;
 
-  exports org.enso.languageserver.boot;
   exports org.enso.languageserver.filemanager to scala.library;
   exports org.enso.languageserver.runtime to scala.library;
   exports org.enso.languageserver.search to scala.library;

engine/language-server/src/main/java/org/enso/languageserver/boot/LanguageServerRunner.java

@@ -8,6 +8,7 @@
 import org.slf4j.event.Level;
 import scala.concurrent.ExecutionContext;
 
+@org.openide.util.lookup.ServiceProvider(service = LanguageServerApi.class)
 public final class LanguageServerRunner extends LanguageServerApi {
   public LanguageServerRunner() {}
 

engine/language-server/src/main/java/org/enso/languageserver/boot/resource/RepoInitialization.java

@@ -1,15 +1,12 @@
 package org.enso.languageserver.boot.resource;
 
 import akka.event.EventStream;
-import java.io.IOException;
-import java.nio.file.FileSystemException;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.util.concurrent.*;
-import org.apache.commons.io.FileUtils;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Semaphore;
 import org.enso.languageserver.data.ProjectDirectoriesConfig;
 import org.enso.languageserver.event.InitializedEvent;
-import org.enso.logger.masking.MaskedPath;
 import org.enso.searcher.memory.InMemorySuggestionsRepo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,14 +69,18 @@ public CompletableFuture<Void> init() {
   }
 
   private CompletableFuture<Void> initSuggestionsRepo() {
-    return CompletableFuture.runAsync(
-            () -> logger.debug("Initializing suggestions repo [{}]...", suggestionsRepo), executor)
-        .thenComposeAsync(
-            v -> {
-              if (!isInitialized)
-                return doInitSuggestionsRepo()
-                    .exceptionallyComposeAsync(this::recoverInitializationError, executor);
-              else return CompletableFuture.completedFuture(v);
+    return CompletableFuture.supplyAsync(
+            () -> {
+              logger.debug("Initializing Suggestions repo [{}]...", suggestionsRepo);
+              try {
+                lock.acquire();
+                if (!isInitialized)
+                  return doInitSuggestionsRepo()
+                      .exceptionallyComposeAsync(this::recoverInitializationError, executor);
+                else return CompletableFuture.completedFuture(null);
+              } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+              }
             },
             executor)
         .thenRunAsync(
@@ -105,60 +106,6 @@ private CompletableFuture<Void> recoverInitializationError(Throwable error) {
         .thenComposeAsync(v -> doInitSuggestionsRepo(), executor);
   }
 
-  private CompletableFuture<Void> clearDatabaseFile(int retries) {
-    return CompletableFuture.runAsync(
-            () -> {
-              if (!isInitialized) {
-                logger.debug("Clear database file. Attempt #{}", retries + 1);
-                try {
-                  Files.delete(projectDirectoriesConfig.suggestionsDatabaseFile().toPath());
-                } catch (IOException e) {
-                  throw new CompletionException(e);
-                }
-              }
-            },
-            executor)
-        .exceptionallyComposeAsync(error -> recoverClearDatabaseFile(error, retries), executor);
-  }
-
-  private CompletableFuture<Void> recoverClearDatabaseFile(Throwable error, int retries) {
-    if (error instanceof CompletionException) {
-      return recoverClearDatabaseFile(error.getCause(), retries);
-    } else if (error instanceof NoSuchFileException) {
-      logger.warn(
-          "Failed to delete the database file. Attempt #{}. File does not exist [{}]",
-          retries + 1,
-          new MaskedPath(projectDirectoriesConfig.suggestionsDatabaseFile().toPath()));
-      return CompletableFuture.completedFuture(null);
-    } else if (error instanceof FileSystemException) {
-      logger.error(
-          "Failed to delete the database file. Attempt #{}. The file will be removed during the"
-              + " shutdown",
-          retries + 1,
-          error);
-      Runtime.getRuntime()
-          .addShutdownHook(
-              new Thread(
-                  () ->
-                      FileUtils.deleteQuietly(projectDirectoriesConfig.suggestionsDatabaseFile())));
-      return CompletableFuture.failedFuture(error);
-    } else if (error instanceof IOException) {
-      logger.error("Failed to delete the database file. Attempt #{}", retries + 1, error);
-      if (retries < MAX_RETRIES) {
-        try {
-          Thread.sleep(RETRY_DELAY_MILLIS);
-        } catch (InterruptedException e) {
-          throw new CompletionException(e);
-        }
-        return clearDatabaseFile(retries + 1);
-      } else {
-        return CompletableFuture.failedFuture(error);
-      }
-    }
-
-    return CompletableFuture.completedFuture(null);
-  }
-
   private CompletionStage<Void> doInitSuggestionsRepo() {
     return FutureConverters.asJava(suggestionsRepo.init()).thenAcceptAsync(res -> {}, executor);
   }

engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala

@@ -55,6 +55,7 @@ import org.enso.profiling.events.NoopEventsMonitor
 import org.enso.searcher.memory.InMemorySuggestionsRepo
 import org.enso.text.{ContentBasedVersioning, Sha3_224VersionCalculator}
 import org.enso.version.BuildVersion
+import com.oracle.truffle.api.TruffleOptions
 import org.graalvm.polyglot.io.MessageEndpoint
 import org.slf4j.event.Level
 import org.slf4j.LoggerFactory
@@ -63,6 +64,7 @@ import java.io.{File, PrintStream}
 import java.net.URI
 import java.nio.charset.StandardCharsets
 import java.time.Clock
+import java.util
 import scala.concurrent.duration.DurationInt
 
 /** A main module containing all components of the server.
@@ -305,7 +307,6 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
   val stdIn     = new ObservablePipedInputStream(stdInSink)
 
   val extraOptions = new java.util.HashMap[String, String]()
-  extraOptions.put(RuntimeServerInfo.ENABLE_OPTION, "true")
   extraOptions.put(RuntimeOptions.INTERACTIVE_MODE, "true")
   extraOptions.put(
     RuntimeOptions.LOG_MASKING,
@@ -317,7 +318,17 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
     Runtime.getRuntime.availableProcessors().toString
   )
 
-  val builder = ContextFactory
+  var extraEngineOptions: util.HashMap[String, String] = null
+  if (TruffleOptions.AOT) {
+    extraEngineOptions = new util.HashMap[String, String]()
+    log.trace("Running Language Server in AOT mode")
+    extraEngineOptions.put(RuntimeServerInfo.ENABLE_OPTION, "true")
+  } else {
+    log.trace("Running Language Server in non-AOT mode")
+    extraOptions.put(RuntimeServerInfo.ENABLE_OPTION, "true")
+  }
+
+  private val builder = ContextFactory
     .create()
     .projectRoot(serverConfig.contentRootPath)
     .logLevel(logLevel)
@@ -328,6 +339,7 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
     .err(stdErr)
     .in(stdIn)
     .options(extraOptions)
+    .engineOptions(extraEngineOptions)
     .messageTransport((uri: URI, peerEndpoint: MessageEndpoint) => {
       if (uri.toString == RuntimeServerInfo.URI) {
         val connection = new RuntimeConnector.Endpoint(

engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala

@@ -202,7 +202,14 @@ class JsonConnectionController(
       )
       mainComponent
         .init()
-        .thenApply(_ => InitializationComponentInitialized.getInstance)
+        .whenComplete((_, ex) =>
+          if (mainComponent.isInitialized) {
+            logger.trace("Resources have been initialized")
+            self ! InitializationComponentInitialized.getInstance()
+          } else {
+            logger.warn("Failed to initialize resources", ex)
+          }
+        )
         .pipeTo(self)
       context.become(initializing(webActor, clientId, req, sender()))
 

engine/language-server/src/main/scala/org/enso/languageserver/runtime/RuntimeConnector.scala

@@ -36,7 +36,7 @@ final class RuntimeConnector(
   override def receive: Receive = {
     case RuntimeConnector.Initialize(engine) =>
       logger.debug(
-        s"Runtime connector established connection with the message endpoint"
+        "Runtime connector established connection with the message endpoint"
       )
       unstashAll()
       context.become(waitingOnEndpoint(engine))
@@ -49,13 +49,15 @@ final class RuntimeConnector(
             Runtime.Api.Response(None, Api.InitializedNotification())
           ) =>
         logger.debug(
-          s"Message endpoint [{}] is initialized. Runtime connector can accept messages",
+          "Message endpoint [{}] is initialized. Runtime connector can accept messages",
           engine
         )
         unstashAll()
         context.become(initialized(engine, Map()))
 
-      case _ => stash()
+      case msg =>
+        logger.trace("Runtime received unexpected message: {}", msg)
+        stash()
     })
 
   /** Performs communication between runtime and language server.
@@ -103,8 +105,8 @@ final class RuntimeConnector(
           handler ! request
         case None =>
           logger.warn(
-            s"No registered handler found for request " +
-            s"[${payload.getClass.getCanonicalName}]"
+            "No registered handler found for request [{}]",
+            payload.getClass.getCanonicalName
           )
       }
 
@@ -120,8 +122,7 @@ final class RuntimeConnector(
         case None =>
           logger.warn(
             "No sender has been found associated with request id [{}], the response [{}] will be dropped",
-            correlationId,
-            payload.getClass.getCanonicalName
+            Array[Any](correlationId, payload.getClass.getCanonicalName)
           )
           payload match {
             case msg: ToLogString =>