Improve performance of finding files to be indexed

CHANGELOG.md

@@ -3,13 +3,19 @@
 ## [Unreleased]
 
 ### Added
+
+### Fixed
+
+## [0.9.10-alpha.2] - 2024-12-05
+
+### Added
+
+* Improve performance of finding files to be indexed
 * Show formatted file path in file not found inspection quickfix name
 * Automatically index bibliography files outside the project that are included by an absolute path
 * Disable quotes inspection when TeX ligatures are disabled by fontspec
 * Inspections can now be suppressed for any single line, or block of text
 
-### Fixed
-
 ## [0.9.9] - 2024-12-01
 
 Welcome to TeXiFy IDEA 0.9.9! This release improves subfiles support, fixes some parser errors and fixes many other small bugs.
@@ -447,7 +453,8 @@ Thanks to @jojo2357 and @MisterDeenis for contributing to this release!
 * Fix some intention previews. ([#2796](https://github.com/Hannah-Sten/TeXiFy-IDEA/issues/2796))
 * Other small bug fixes and improvements. ([#2776](https://github.com/Hannah-Sten/TeXiFy-IDEA/issues/2776), [#2774](https://github.com/Hannah-Sten/TeXiFy-IDEA/issues/2774), [#2765](https://github.com/Hannah-Sten/TeXiFy-IDEA/issues/2765)-[#2773](https://github.com/Hannah-Sten/TeXiFy-IDEA/issues/2773))
 
-[Unreleased]: https://github.com/Hannah-Sten/TeXiFy-IDEA/compare/v0.9.9...HEAD
+[Unreleased]: https://github.com/Hannah-Sten/TeXiFy-IDEA/compare/v0.9.10-alpha.2...HEAD
+[0.9.10-alpha.2]: https://github.com/Hannah-Sten/TeXiFy-IDEA/compare/v0.9.9...v0.9.10-alpha.2
 [0.9.9]: https://github.com/Hannah-Sten/TeXiFy-IDEA/compare/v0.9.8...v0.9.9
 [0.9.8]: https://github.com/Hannah-Sten/TeXiFy-IDEA/compare/v0.9.7...v0.9.8
 [0.9.7]: https://github.com/Hannah-Sten/TeXiFy-IDEA/compare/v0.9.6...v0.9.7

gradle.properties

@@ -1,4 +1,4 @@
-pluginVersion = 0.9.9
+pluginVersion = 0.9.10-alpha.2
 
 #     Info about build ranges: https://www.jetbrains.org/intellij/sdk/docs/basics/getting_started/build_number_ranges.html
 #    Note that an xyz branch corresponds to version 20xy.z and a since build of xyz.*

src/nl/hannahsten/texifyidea/index/file/LatexIndexableSetContributor.kt

@@ -75,19 +75,29 @@ class LatexIndexableSetContributor : IndexableSetContributor() {
         // Using the index while building it may be problematic, cache the result and hope it doesn't create too much trouble
         if (Cache.externalDirectFileInclusions == null) {
             if (!DumbService.isDumb(project)) {
-                // For now, just do this for bibliography and direct input commands, as there this is most common
-                val externalFiles = LatexIncludesIndex.Util.getCommandsByNames(CommandMagic.includeOnlyExtensions.entries.filter { it.value.contains("bib") || it.value.contains("tex") }.map { it.key }.toSet(), project, GlobalSearchScope.projectScope(project))
-                    // We can't add single files, so take the parent
-                    .mapNotNull {
-                        val path = it.requiredParameter(0) ?: return@mapNotNull null
-                        if (File(path).isAbsolute) {
-                            LocalFileSystem.getInstance().findFileByPath(path)?.parent
-                        }
-                        else {
-                            it.containingFile.parent?.virtualFile?.findFileByRelativePath(path)?.parent
+                try {
+                    // For now, just do this for bibliography and direct input commands, as there this is most common
+                    val externalFiles = LatexIncludesIndex.Util.getCommandsByNames(CommandMagic.includeOnlyExtensions.entries.filter { it.value.contains("bib") || it.value.contains("tex") }.map { it.key }.toSet(), project, GlobalSearchScope.projectScope(project))
+                        // We can't add single files, so take the parent
+                        .mapNotNull {
+                            val path = it.requiredParameter(0) ?: return@mapNotNull null
+                            if (File(path).isAbsolute) {
+                                LocalFileSystem.getInstance().findFileByPath(path)?.parent
+                            }
+                            else {
+                                it.containingFile.parent?.virtualFile?.findFileByRelativePath(path)?.parent
+                            }
                         }
+                    Cache.externalDirectFileInclusions = externalFiles.toSet()
+                } catch (e: Throwable) {
+                    // This is very rare, but it can happen, in which case we will ignore and try again later
+                    if (e.message?.contains("Indexing process should not rely on non-indexed file data") == true) {
+                        Log.warn("Ignored index not ready: " + e.message)
                     }
-                Cache.externalDirectFileInclusions = externalFiles.toSet()
+                    else {
+                        throw e
+                    }
+                }
             }
         }
         roots.addAll(Cache.externalDirectFileInclusions?.filter { it.exists() } ?: emptyList())

src/nl/hannahsten/texifyidea/settings/sdk/TexliveSdk.kt

@@ -61,7 +61,7 @@ open class TexliveSdk(name: String = "TeX Live SDK") : LatexSdk(name) {
         if (!paths.isNullOrEmpty()) {
             for (path in paths.split("\\s+".toRegex())) {
                 // Resolve symlinks
-                val resolvedPath = runCommand("readlink", "-f", path) ?: path
+                val resolvedPath = if (!SystemInfo.isWindows) runCommand("readlink", "-f", path) ?: path else path
 
                 // We don't know for sure whether this path contains 'texlive':
                 // e.g. C:\texnolive\2021\bin\pdflatex.exe can be perfectly valid

src/nl/hannahsten/texifyidea/util/CommandRunner.kt

@@ -38,6 +38,14 @@
     try {
         Log.debug("isEDT=${SwingUtilities.isEventDispatchThread()} Executing in ${workingDirectory ?: "current working directory"} ${GeneralCommandLine(*commands).commandLineString}")
 
+        // where/which commands occur often but do not change since the output depends on PATH, so can be cached
+        val isExecutableLocationCommand = commands.size == 2 && listOf("where", "which").contains(commands[0])
+        if (isExecutableLocationCommand && SystemEnvironment.executableLocationCache.get(commands[1]) != null) {
+            val standardOutput = SystemEnvironment.executableLocationCache[commands[1]]
+            Log.debug("Retrieved output of $commands from cache: $standardOutput")
+            return@withContext CommandResult(0, standardOutput, null)
+        }
+
         val processBuilder = GeneralCommandLine(*commands)
             .withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.CONSOLE)
             .withWorkDirectory(workingDirectory)
@@ -59,7 +67,7 @@
                         it.readText()
                     }
                     catch (e: IOException) {
-                        // In some case directly after IDE start, the stream may be closed already, so ignore that
+                        // In some cases directly after IDE start (after a timeout?), the stream may be closed already, so ignore that
                         if (e.message?.contains("Stream closed") == true) {
                             Log.info("Ignored closed stream: " + e.message)
                             e.message
@@ -81,6 +89,11 @@
         val result = CommandResult(process.awaitExit(), output?.await()?.trim(), error?.await()?.trim())
         Log.debug("${commands.firstOrNull()} exited with ${result.exitCode} ${result.standardOutput?.take(100)} ${result.errorOutput?.take(100)}")
 
+        // Update cache of where/which output
+        if (isExecutableLocationCommand) {
+            SystemEnvironment.executableLocationCache[commands[1]] = result.standardOutput
+        }
+
         return@withContext result
     }
     catch (e: IOException) {

src/nl/hannahsten/texifyidea/util/SystemEnvironment.kt

@@ -52,6 +52,10 @@ class SystemEnvironment {
             return isAvailable
         }
 
+        // Map a system command to the full location, as raw output (so may be multiple paths on Windows) from the where/which command
+        // This is cached because the value will not change between restarts
+        var executableLocationCache = mutableMapOf<String, String?>()
+
         // Assumes version will be given in the format GNOME Document Viewer 3.34.2
         val evinceVersion: DefaultArtifactVersion by lazy {
             DefaultArtifactVersion("evince --version".runCommand()?.split(" ")?.lastOrNull() ?: "")