Refactor in accordance to cursorless issue 1488 specs

.github/workflows/build-and-publish.yml

@@ -6,20 +6,21 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: pnpm/action-setup@v2
-      - name: Install Node.js with pnpm
-        uses: actions/setup-node@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.0.0
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: "pnpm"
       - run: pnpm install
 
-      - uses: mymindstorm/setup-emsdk@v11
+      - uses: mymindstorm/setup-emsdk@v14
         with:
-          version: 2.0.24
+          version: 3.1.74
 
       - run: pnpm build
 
       - uses: JS-DevTools/npm-publish@v1
         with:
-          token: ${{ secrets.NPM_TOKEN }}
+          token: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -1,2 +1,3 @@
 out/
-node_modules/
+node_modules/
+.DS_Store

build.ts

@@ -16,11 +16,51 @@ const outDir = path.join(__dirname, "out");
 
 let hasErrors = false;
 
+async function gitCloneOverload(name: string, repoUrl: string, useLatest: boolean, {commitHash}: {commitHash?: string}) {
+  let packagePath;
+  try {
+    packagePath = findRoot(require.resolve(name));
+  } catch (_) {
+    packagePath = path.join(__dirname, "node_modules", name);
+  }
+
+  let commitInfo = (commitHash === undefined) ? "latest" : commitHash;
+
+  try {
+    console.log(`🗑️  Deleting cached node dep for ${ name }`);
+    await exec(`rm -rf ${ packagePath }`)
+    console.log(`⬇️  Cloning ${ name } from git (${commitInfo})`);
+    await exec(`git clone ${ repoUrl } ${ packagePath }`)
+    if (!useLatest) {
+      if (commitHash !== undefined) {
+        process.chdir(packagePath);
+        await exec(`git reset --hard ${ commitHash }`);
+      } else throw new Error("Latest commit is not being used, yet no commit hash was specified");
+    }
+  } catch (err) {
+      console.error(`❗Failed to clone git repo for ${ name }:\n`, err);
+  }
+}
+
 async function buildParserWASM(
   name: string,
-  { subPath, generate }: { subPath?: string; generate?: boolean } = {}
+  { subPath, generate }: 
+  { subPath?: string; generate?: boolean } = {}
 ) {
+
   const label = subPath ? path.join(name, subPath) : name;
+
+  let cliPackagePath;
+  try {
+    cliPackagePath = findRoot(require.resolve("tree-sitter-cli"));
+  } catch(_) {
+    cliPackagePath = path.join(__dirname, "node_modules", "tree-sitter-cli");
+  }
+
+  let cliPath = path.join(cliPackagePath, "tree-sitter");
+  let generateCommand = cliPath.concat(" generate");
+  let buildCommand = cliPath.concat(" build --wasm");
+
   try {
     console.log(`⏳ Building ${label}`);
     let packagePath;
@@ -31,9 +71,10 @@ async function buildParserWASM(
     }
     const cwd = subPath ? path.join(packagePath, subPath) : packagePath;
     if (generate) {
-      await exec(`pnpm tree-sitter generate`, { cwd });
+      await exec(generateCommand, { cwd });
     }
-    await exec(`pnpm tree-sitter build-wasm ${cwd}`);
+    await exec(buildCommand, { cwd }); 
+    await exec(`mv *.wasm ${outDir}`, { cwd });
     console.log(`✅ Finished building ${label}`);
   } catch (e) {
     console.error(`🔥 Failed to build ${label}:\n`, e);
@@ -50,30 +91,48 @@ fs.mkdirSync(outDir);
 process.chdir(outDir);
 
 const grammars = Object.keys(packageInfo.devDependencies)
-  .filter((n) => n.startsWith("tree-sitter-") && n !== "tree-sitter-cli")
-  .concat('@tree-sitter-grammars/tree-sitter-zig')
-  .concat("@tlaplus/tree-sitter-tlaplus")
+  .filter((n) => n.startsWith("tree-sitter-") && n !== "tree-sitter-cli" && n !== "tree-sitter")
   .filter((s) => !langArg || s.includes(langArg));
 
 PromisePool.withConcurrency(os.cpus().length)
   .for(grammars)
-  .process(async (name) => {
-    if (name == "tree-sitter-rescript") {
-      await buildParserWASM(name, { generate: true });
-    } else if (name == "tree-sitter-ocaml") {
-      await buildParserWASM(name, { subPath: "ocaml" });
+  .process(async (name : string) => {
+    if (name == "tree-sitter-agda") {
+      await gitCloneOverload(name, "https://github.com/tree-sitter/tree-sitter-agda.git", false, {commitHash: "47802091de0cb8ac2533d67ac37e65692c5902c4"});
+      await buildParserWASM(name)
+    } else if (name == "tree-sitter-perl") {
+      await gitCloneOverload(name, "https://github.com/tree-sitter-perl/tree-sitter-perl.git", false, {commitHash: "b467c41aed6e52c7f581c18a3b50821995a82c95"});
+      await buildParserWASM(name, {generate: true});
     } else if (name == "tree-sitter-php") {
       await buildParserWASM(name, { subPath: "php" });
     } else if (name == "tree-sitter-typescript") {
       await buildParserWASM(name, { subPath: "typescript" });
       await buildParserWASM(name, { subPath: "tsx" });
-    } else {
+    } else if (name == "tree-sitter-latex") {
+      await buildParserWASM(name, {generate: true});
+    } else if (name == "tree-sitter-xml") {
+      await buildParserWASM(name, {subPath: "xml"});
+      await buildParserWASM(name, {subPath: "dtd"});
+    } else if (name == "tree-sitter-query") {
+      await gitCloneOverload(name, "https://github.com/tree-sitter-grammars/tree-sitter-query.git", false, {commitHash: "a6674e279b14958625d7a530cabe06119c7a1532"});
+      await buildParserWASM(name);
+    } else if (name == "tree-sitter-elixir") {
+      await gitCloneOverload(name, "https://github.com/elixir-lang/tree-sitter-elixir.git", false, {commitHash: "02a6f7fd4be28dd94ee4dd2ca19cb777053ea74e"});
+      await buildParserWASM(name);
+    } else if (name == "tree-sitter-markdown") {
+      await gitCloneOverload(name, "https://github.com/tree-sitter-grammars/tree-sitter-markdown", false, {commitHash: "192407ab5a24bfc24f13332979b5e7967518754a"});
+      await buildParserWASM(name, {subPath: "tree-sitter-markdown"});
+      await buildParserWASM(name, {subPath: "tree-sitter-markdown-inline"});
+    }
+    else {
       await buildParserWASM(name);
     }
   })
   .then(async () => {
     if (hasErrors) {
+      //not sure if this failsafe is actually required, but it doesn't hurt to ensure that no wasms can be published from a failed build 
+      fs.rmSync(outDir, { recursive: true, force: true });
+      fs.mkdirSync(outDir);
       process.exit(1);
     }
-    await exec(`mv *.wasm ${outDir}`, { cwd: __dirname });
-  });
+  });

package.json

@@ -1,63 +1,73 @@
 {
-  "name": "tree-sitter-wasms",
-  "version": "0.1.12",
+  "name": "@cursorless/tree-sitter-wasms",
+  "version": "0.2.0",
   "description": "Prebuilt WASM binaries for tree-sitter's language parsers.",
-  "repository": "https://github.com/Gregoor/tree-sitter-wasms",
-  "author": "Gregor <[email protected]>, Menci <[email protected]>",
+  "repository": "https://github.com/cursorless-dev/tree-sitter-wasms",
+  "author": "Gregor <[email protected]>, Menci <[email protected]>, rinOfTheStars",
   "license": "Unlicense",
   "scripts": {
     "build": "ts-node build.ts"
   },
   "devDependencies": {
     "@supercharge/promise-pool": "^3.1.1",
-    "@tlaplus/tree-sitter-tlaplus": "^1.2.4",
-    "@tree-sitter-grammars/tree-sitter-zig": "^1.0.0",
     "@types/find-root": "^1.1.4",
     "@types/node": "^20.11.17",
-    "@willjouo/tree-sitter-r": "^0.0.1",
     "find-root": "^1.1.0",
-    "tree-sitter-bash": "^0.20.5",
-    "tree-sitter-c": "^0.20.7",
-    "tree-sitter-c-sharp": "^0.20.0",
-    "tree-sitter-cli": "^0.20.8",
-    "tree-sitter-cpp": "^0.20.4",
-    "tree-sitter-css": "^0.20.0",
-    "tree-sitter-elisp": "^1.3.0",
-    "tree-sitter-elixir": "^0.1.1",
-    "tree-sitter-elm": "^4.5.0",
-    "tree-sitter-embedded-template": "^0.20.0",
-    "tree-sitter-go": "^0.20.0",
-    "tree-sitter-html": "^0.20.0",
-    "tree-sitter-java": "^0.20.2",
+    "tree-sitter": "^0.22.4",
+    "tree-sitter-agda": "github:tree-sitter/tree-sitter-agda#47802091de0cb8ac2533d67ac37e65692c5902c4",
+    "tree-sitter-bash": "^0.23.3",
+    "tree-sitter-c": "^0.23.4",
+    "tree-sitter-c-sharp": "^0.23.1",
+    "tree-sitter-cli": "^0.24.7",
+    "tree-sitter-clojure": "github:sogaiu/tree-sitter-clojure#f4236d4da8aa92bc105d9c118746474c608e6af7",
+    "tree-sitter-cpp": "^0.23.4",
+    "tree-sitter-css": "^0.23.1",
+    "tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#e81af6ab94a728ed99c30083be72d88e6d56cf9e",
+    "tree-sitter-elixir": "github:elixir-lang/tree-sitter-elixir#02a6f7fd4be28dd94ee4dd2ca19cb777053ea74e",
+    "tree-sitter-elm": "github:elm-tooling/tree-sitter-elm#e34bdc5c512918628b05b48e633f711123204e45",
+    "tree-sitter-gdscript": "github:PrestonKnopp/tree-sitter-gdscript#48b49330888a4669b48619b211cc8da573827725",
+    "tree-sitter-gleam": "github:gleam-lang/tree-sitter-gleam#066704e4826699e754d351e3bbe12bf2e51de9d8",
+    "tree-sitter-go": "^0.23.4",
+    "tree-sitter-haskell": "^0.23.1",
+    "tree-sitter-hcl": "github:tree-sitter-grammars/tree-sitter-hcl#9e3ec9848f28d26845ba300fd73c740459b83e9b",
+    "tree-sitter-html": "^0.23.2",
+    "tree-sitter-java": "^0.23.2",
     "tree-sitter-javascript": "^0.20.3",
-    "tree-sitter-json": "^0.20.2",
-    "tree-sitter-kotlin": "^0.3.1",
+    "tree-sitter-json": "^0.24.8",
+    "tree-sitter-julia": "^0.23.1",
+    "tree-sitter-kotlin": "^0.3.8",
+    "tree-sitter-latex": "github:latex-lsp/tree-sitter-latex#7b06f6ed394308e7407a1703d2724128c45fc9d7",
     "tree-sitter-lua": "^2.1.3",
-    "tree-sitter-objc": "^2.1.0",
-    "tree-sitter-ocaml": "^0.20.4",
-    "tree-sitter-php": "^0.22.0",
-    "tree-sitter-python": "^0.21.0",
-    "tree-sitter-ql": "^1.0.0",
-    "tree-sitter-rescript": "github:rescript-lang/tree-sitter-rescript#6376fa028f31aa4e26ca2c8f007e322cd2a5eb4a",
-    "tree-sitter-ruby": "^0.20.1",
-    "tree-sitter-rust": "^0.20.4",
-    "tree-sitter-scala": "^0.19.0",
-    "tree-sitter-solidity": "github:JoranHonig/tree-sitter-solidity#b239a95f94cfcc6e7b3e961bc73a28d55e214f02",
-    "tree-sitter-swift": "^0.4.0",
-    "tree-sitter-systemrdl": "^0.7.0",
-    "tree-sitter-toml": "^0.5.1",
-    "tree-sitter-typescript": "^0.20.5",
-    "tree-sitter-vue": "https://github.com/tree-sitter-grammars/tree-sitter-vue#7e48557b903a9db9c38cea3b7839ef7e1f36c693",
-    "tree-sitter-yaml": "^0.5.0",
+    "tree-sitter-markdown": "github:tree-sitter-grammars/tree-sitter-markdown#192407ab5a24bfc24f13332979b5e7967518754a",
+    "tree-sitter-nix": "github:nix-community/tree-sitter-nix#21897cc3dcd15325303e46b85295b743742af6ab",
+    "tree-sitter-perl": "github:tree-sitter-perl/tree-sitter-perl#b467c41aed6e52c7f581c18a3b50821995a82c95",
+    "tree-sitter-php": "^0.23.11",
+    "tree-sitter-python": "^0.23.6",
+    "tree-sitter-query": "github:tree-sitter-grammars/tree-sitter-query#a6674e279b14958625d7a530cabe06119c7a1532",
+    "tree-sitter-ruby": "^0.23.1",
+    "tree-sitter-rust": "^0.23.2",
+    "tree-sitter-scala": "^0.23.4",
+    "tree-sitter-scss": "github:tree-sitter-grammars/tree-sitter-scss#00a4cd98d5fc71aa5a55174e7ed5e34c9cc85a87",
+    "tree-sitter-sparql": "github:IoannisNezis/tree-sitter-sparql#3c468620e7c2ffa69441de60161c4e4a919969f7",   
+    "tree-sitter-swift": "^0.6.0",
+    "tree-sitter-talon": "github:wenkokke/tree-sitter-talon#7119d7f3fefd0b0c7e900120b51f85a23e6c3160",
+    "tree-sitter-typescript": "^0.23.2",
+    "tree-sitter-xml": "github:tree-sitter-grammars/tree-sitter-xml#ce150c5be7f617e18ffa2064213e5d9c4cb5d69e",
     "ts-node": "^10.9.2",
-    "typescript": "5.3.3"
+    "typescript": "^5.7.3"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+        "tree-sitter-cli"
+    ]
   },
   "files": [
     "/out"
   ],
-  "packageManager": "pnpm@8.10.5",
+  "packageManager": "pnpm@10.0.0",
   "main": "bindings/node",
   "dependencies": {
-    "tree-sitter-wasms": "^0.1.11"
+    "tree-sitter": "^0.22.4",
+    "tree-sitter-cli": "^0.24.7"
   }
 }