feat: support new treesitter syntax highlighting

.ncurc.json

@@ -10,6 +10,7 @@
     "@vscode/proxy-agent",
     "@vscode/ripgrep",
     "@vscode/spdlog",
+    "@vscode/tree-sitter-wasm",
     "@vscode/vscode-languagedetection",
     "@vscode/windows-process-tree",
     "@vscode/windows-registry",

rollup/rollup.config.ts

@@ -581,7 +581,14 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
       }),
       {
         name: 'dynamic-import-polyfill',
-        renderDynamicImport (): { left: string, right: string } {
+        renderDynamicImport ({ targetModuleId }): { left: string, right: string } {
+          // Hack for @vscode/tree-sitter-wasm that doesn't export its parser correctly (as default instead of a named export, in commonjs)
+          if (targetModuleId === '@vscode/tree-sitter-wasm') {
+            return {
+              left: 'import(',
+              right: ').then(module => ({ Parser: module.default ?? module }))'
+            }
+          }
           // dynamic imports of vscode-oniguruma and vscode-textmate aren't working without it on vite
           return {
             left: 'import(',

scripts/install-vscode

@@ -91,7 +91,7 @@ mkdir -p $output_directory
 echo "Building vscode..."
 NODE_OPTIONS=--max-old-space-size=8192 npx tsc --declaration --importHelpers --module es2020 --preserveConstEnums --outDir "$output_directory/src" --rootDir .
 # Copy files that are already built and assets
-find ./ \( -name '*.js' -o -name '*.d.ts' -o -name '*.ttf' -o -name '*.css' -o -name '*.mp3' -o -name '*.svg' -o -name '*.png' -o -name '*.html' -o -name '*.sh' -o -name '*.zsh' -o -name '*.ps1' \) -exec rsync -R \{\} "$output_directory/src" \;
+find ./ \( -name '*.js' -o -name '*.d.ts' -o -name '*.ttf' -o -name '*.css' -o -name '*.mp3' -name '*.scm' -o -name '*.svg' -o -name '*.png' -o -name '*.html' -o -name '*.sh' -o -name '*.zsh' -o -name '*.ps1' \) -exec rsync -R \{\} "$output_directory/src" \;
 
 cd ..
 cp package.json $output_directory

src/service-override/treesitter.ts

@@ -0,0 +1,21 @@
+import { IEditorOverrideServices } from 'vs/editor/standalone/browser/standaloneServices'
+import { SyncDescriptor } from 'vs/platform/instantiation/common/descriptors'
+import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'
+import { TreeSitterTextModelService } from 'vs/editor/browser/services/treeSitter/treeSitterParserService'
+import { ITreeSitterTokenizationFeature } from 'vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.service'
+import { TreeSitterTokenizationFeature } from 'vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature'
+import { registerAssets } from '../assets'
+import 'vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution'
+
+registerAssets({
+  'vs/../../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter.wasm': new URL('@vscode/tree-sitter-wasm/wasm/tree-sitter.wasm', import.meta.url).href,
+  'vs/../../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter-typescript.wasm': new URL('@vscode/tree-sitter-wasm/wasm/tree-sitter-typescript.wasm', import.meta.url).href,
+  'vs/editor/common/languages/highlights/typescript.scm': new URL('../../vscode/src/vs/editor/common/languages/highlights/typescript.scm', import.meta.url).href
+})
+
+export default function getServiceOverride (): IEditorOverrideServices {
+  return {
+    [ITreeSitterParserService.toString()]: new SyncDescriptor(TreeSitterTextModelService, [], false),
+    [ITreeSitterTokenizationFeature.toString()]: new SyncDescriptor(TreeSitterTokenizationFeature, [], false)
+  }
+}