Better split service overrides

demo/tsconfig.json

@@ -16,6 +16,7 @@
     "noImplicitReturns": true,
     "baseUrl": ".",
     "paths": {
+      "vscode/vscode/*": ["../dist/main/vscode/src/*"],
       "vscode/*": ["../dist/main/*"]
     }
   },

demo/vite.config.ts

@@ -71,8 +71,7 @@ export default defineConfig({
       '@codingame/monaco-vscode-typescript-language-features-default-extension', '@codingame/monaco-vscode-markdown-language-features-default-extension',
       '@codingame/monaco-vscode-json-language-features-default-extension', '@codingame/monaco-vscode-css-language-features-default-extension',
       '@codingame/monaco-vscode-npm-default-extension', '@codingame/monaco-vscode-css-default-extension', '@codingame/monaco-vscode-markdown-basics-default-extension', '@codingame/monaco-vscode-html-default-extension',
-      '@codingame/monaco-vscode-html-language-features-default-extension', '@codingame/monaco-vscode-configuration-editing-default-extension', '@codingame/monaco-vscode-media-preview-default-extension', '@codingame/monaco-vscode-markdown-math-default-extension',
-      'vscode/workers/textmate.worker'
+      '@codingame/monaco-vscode-html-language-features-default-extension', '@codingame/monaco-vscode-configuration-editing-default-extension', '@codingame/monaco-vscode-media-preview-default-extension', '@codingame/monaco-vscode-markdown-math-default-extension'
     ],
     esbuildOptions: {
       plugins: [{

package.json

@@ -15,7 +15,7 @@
   },
   "type": "module",
   "scripts": {
-    "build": "npm run clean && npm run generate-types && npm run lint && npm run compile-rollup-plugins && npm run compile-treemending-script && npm run compile && npm run compile-server && npm run compile-default-extensions && npm run copy-monaco-editor",
+    "build": "npm run clean && npm run lint && npm run compile && npm run compile-rollup-plugins && npm run generate-types && npm run compile-treemending-script && npm run compile-server && npm run compile-default-extensions && npm run copy-monaco-editor",
     "compile": "NODE_OPTIONS=--max_old_space_size=8192 rollup --config rollup/rollup.config.ts --configPlugin 'typescript={tsconfig: `tsconfig.rollup-config.json`}' --vscode-version ${npm_package_config_vscode_version} --vscode-ref ${npm_package_config_vscode_ref}",
     "compile-default-extensions": "NODE_OPTIONS=--max_old_space_size=8192 rollup --config rollup/rollup.default-extensions.ts --configPlugin 'typescript={tsconfig: `tsconfig.rollup-config-default-extensions.json`}'",
     "clean": "rm -rf dist/",

rollup/rollup-metadata-plugin.ts

@@ -1,33 +1,59 @@
-import type { OutputBundle, OutputOptions, Plugin, PluginContext } from 'rollup'
+import type { OutputBundle, OutputChunk, OutputOptions, Plugin, PluginContext } from 'rollup'
 import { builtinModules } from 'module'
+import path from 'path'
 
 interface Group {
   name: string
   dependencies: Set<string>
+  modules: Set<string>
   entrypoints: Set<string>
 }
 
 interface Options {
+  stage?: 'generateBundle' | 'writeBundle'
   getGroup?: (entryPoint: string, options: OutputOptions) => string
-  handle (this: PluginContext, groupName: string, dependencies: Set<string>, entrypoints: Set<string>, options: OutputOptions, bundle: OutputBundle): void | Promise<void>
+  handle (this: PluginContext, groupName: string, dependencies: Set<string>, entrypoints: Set<string>, exclusiveModules: Set<string>, options: OutputOptions, bundle: OutputBundle): void | Promise<void>
 }
 
-export default ({ handle, getGroup = () => 'main' }: Options): Plugin => ({
+export default ({ handle, getGroup = () => 'main', stage = 'generateBundle' }: Options): Plugin => ({
   name: 'generate-metadata',
-  async generateBundle (options: OutputOptions, bundle: OutputBundle) {
-    const externalDependencyCache = new Map<string, Set<string>>()
-    const getModuleExternalDependencies = (id: string, path: string[]): Set<string> => {
-      if (!externalDependencyCache.has(id)) {
+  [stage]: async function (this: PluginContext, options: OutputOptions, bundle: OutputBundle) {
+    const dependencyCache = new Map<string, { externals: Set<string>, internals: Set<string> }>()
+
+    const inputToOutput = Object.fromEntries(Object.values(bundle).filter((chunk): chunk is OutputChunk => (chunk as OutputChunk).code != null).map(chunk => {
+      return [
+        chunk.facadeModuleId,
+        path.resolve(options.dir!, chunk.preliminaryFileName)
+      ]
+    }))
+    const getModuleDependencies = (id: string, path: string[]): { externals: Set<string>, internals: Set<string> } => {
+      if (path.includes(id)) {
+        // Break recursive imports
+        return {
+          externals: new Set(),
+          internals: new Set()
+        }
+      }
+      if (!dependencyCache.has(id)) {
         const moduleInfo = this.getModuleInfo(id)!
         if (moduleInfo.isExternal) {
           const match = /^(?:@[^/]*\/)?[^/]*/.exec(id)
-          externalDependencyCache.set(id, new Set(match != null && !builtinModules.includes(match[0]) ? [match[0]] : []))
+          dependencyCache.set(id, {
+            externals: new Set(match != null && !builtinModules.includes(match[0]) ? [match[0]] : []),
+            internals: new Set([])
+          })
         } else {
-          externalDependencyCache.set(id, new Set([...moduleInfo.importedIds, ...moduleInfo.dynamicallyImportedIds].flatMap(depId => Array.from(getModuleExternalDependencies(depId, [...path, id])))))
+          const dependencies = [...moduleInfo.importedIds, ...moduleInfo.dynamicallyImportedIds].map(depId => {
+            return getModuleDependencies(depId, [...path, id])
+          })
+          dependencyCache.set(id, {
+            externals: new Set(dependencies.map(t => t.externals).flatMap(map => Array.from(map))),
+            internals: new Set([inputToOutput[id] ?? id, ...dependencies.map(t => t.internals).flatMap(map => Array.from(map))])
+          })
         }
       }
 
-      return externalDependencyCache.get(id)!
+      return dependencyCache.get(id)!
     }
 
     const groups = new Map<string, Group>()
@@ -37,22 +63,32 @@
         continue
       }
       const groupName = getGroup(id, options)
-      const externalDependencies = getModuleExternalDependencies(moduleInfo.id, [])
+      const dependencies = getModuleDependencies(moduleInfo.id, [])
 
       if (!groups.has(groupName)) {
         groups.set(groupName, {
           dependencies: new Set<string>(),
           entrypoints: new Set<string>(),
-          name: groupName
+          name: groupName,
+          modules: new Set<string>()
         })
       }
       const group = groups.get(groupName)!
-      externalDependencies.forEach(d => group.dependencies.add(d))
+      dependencies.externals.forEach(d => group.dependencies.add(d))
+      dependencies.internals.forEach(d => group.modules.add(d))
       group.entrypoints.add(id)
     }
 
+    const moduleUseCount = new Map<string, number>()
+    for (const [_, group] of groups.entries()) {
+      for (const module of group.modules) {
+        moduleUseCount.set(module, (moduleUseCount.get(module) ?? 0) + 1)
+      }
+    }
+
     await Promise.all(Array.from(groups.entries()).map(async ([name, group]) => {
-      await handle.call(this, name, group.dependencies, group.entrypoints, options, bundle)
+      const exclusiveModules = new Set(Array.from(group.modules).filter(module => moduleUseCount.get(module)! <= 1))
+      await handle.call(this, name, group.dependencies, exclusiveModules, group.entrypoints, options, bundle)
     }))
   }
 })

rollup/rollup.config.ts

@@ -111,6 +111,8 @@ const BASE_DIR = path.resolve(__dirname, '..')
 const TSCONFIG = path.resolve(BASE_DIR, 'tsconfig.rollup.json')
 const SRC_DIR = path.resolve(BASE_DIR, 'src')
 const DIST_DIR = path.resolve(BASE_DIR, 'dist')
+const DIST_DIR_MAIN = path.resolve(DIST_DIR, 'main')
+const VSCODE_SRC_DIST_DIR = path.resolve(DIST_DIR_MAIN, 'vscode', 'src')
 const VSCODE_DIR = path.resolve(BASE_DIR, 'vscode')
 const VSCODE_SRC_DIR = path.resolve(VSCODE_DIR, 'src')
 const NODE_MODULES_DIR = path.resolve(BASE_DIR, 'node_modules')
@@ -755,7 +757,11 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
       chunkFileNames: '[name].js',
       hoistTransitiveImports: false
     }],
-    plugins: [{
+    plugins: [importMetaAssets({
+      include: ['**/*.ts', '**/*.js'],
+      // assets are externals and this plugin is not able to ignore external assets
+      exclude: ['**/service-override/textmate.js', '**/service-override/languageDetectionWorker.js']
+    }), {
       name: 'improve-treeshaking',
       transform (code) {
         const ast = recast.parse(code, {
@@ -836,7 +842,7 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
         }
         return 'main'
       },
-      async handle (groupName, dependencies, entrypoints, options) {
+      async handle (groupName, dependencies, exclusiveModules, entrypoints, options, bundle) {
         if (groupName === 'main') {
           // Generate package.json
           const packageJson: PackageJson = {
@@ -856,6 +862,14 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
                 types: './extensions.d.ts',
                 default: './extensions.js'
               },
+              './assets': {
+                types: './assets.d.ts',
+                default: './assets.js'
+              },
+              './lifecycle': {
+                types: './lifecycle.d.ts',
+                default: './lifecycle.js'
+              },
               './service-override/*': {
                 types: './service-override/*.d.ts',
                 default: './service-override/*.js'
@@ -866,6 +880,9 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
               './monaco': {
                 types: './monaco.d.ts',
                 default: './monaco.js'
+              },
+              './vscode/*': {
+                default: './vscode/src/*.js'
               }
             },
             typesVersions: {
@@ -881,6 +898,15 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
                 ],
                 monaco: [
                   './monaco.d.ts'
+                ],
+                assets: [
+                  './assets.d.ts'
+                ],
+                lifecycle: [
+                  './lifecycle.d.ts'
+                ],
+                'vscode/*': [
+                  './vscode/src/*.d.ts'
                 ]
               }
             },
@@ -921,27 +947,7 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
               ...Object.fromEntries(Object.entries(pkg.dependencies).filter(([key]) => dependencies.has(key)))
             }
           }
-
-          const reexportFrom = `vscode/${path.relative(options.dir!, serviceOverrideEntryPoint).slice(0, -3)}`
-
-          const entrypointInfo = this.getModuleInfo(serviceOverrideEntryPoint)!
-          const codeLines: string[] = []
-          if ((entrypointInfo.exports ?? []).includes('default')) {
-            codeLines.push(`export { default } from '${reexportFrom}'`)
-          }
-          if ((entrypointInfo.exports ?? []).some(e => e !== 'default')) {
-            codeLines.push(`export * from '${reexportFrom}'`)
-          }
-          if ((entrypointInfo.exports ?? []).length === 0) {
-            codeLines.push(`import '${reexportFrom}'`)
-          }
-          await fsPromise.writeFile(path.resolve(directory, 'index.js'), codeLines.join('\n'))
-          await fsPromise.writeFile(path.resolve(directory, 'index.d.ts'), codeLines.join('\n'))
-
           if (workerEntryPoint != null) {
-            const workerFrom = `vscode/${path.relative(options.dir!, workerEntryPoint).slice(0, -3)}`
-            await fsPromise.writeFile(path.resolve(directory, 'worker.js'), `import '${workerFrom}'`)
-
             packageJson.exports = {
               '.': {
                 default: './index.js'
@@ -952,10 +958,123 @@ export default (args: Record<string, string>): rollup.RollupOptions[] => {
             }
           }
 
-          await fsPromise.writeFile(path.resolve(directory, 'package.json'), JSON.stringify(packageJson, null, 2))
+          const entrypointInfo = this.getModuleInfo(serviceOverrideEntryPoint)!
+
+          const groupBundle = await rollup.rollup({
+            input: {
+              index: 'entrypoint',
+              ...(workerEntryPoint != null
+                ? {
+                    worker: 'worker'
+                  }
+                : {})
+            },
+            external,
+            plugins: [
+              importMetaAssets({
+                include: ['**/*.ts', '**/*.js'],
+                // assets are externals and this plugin is not able to ignore external assets
+                exclude: ['**/service-override/textmate.js', '**/service-override/languageDetectionWorker.js']
+              }),
+              nodeResolve({
+                extensions: EXTENSIONS
+              }), {
+                name: 'loader',
+                resolveId (source, importer) {
+                  if (source === 'entrypoint' || source === 'worker') {
+                    return source
+                  }
+                  if (source.startsWith('monaco-editor/')) {
+                    return null
+                  }
+                  const importerDir = path.dirname(path.resolve(DIST_DIR_MAIN, importer ?? '/'))
+                  const resolved = path.resolve(importerDir, source)
+                  const resolvedWithExtension = resolved.endsWith('.js') ? resolved : `${resolved}.js`
+
+                  const isNotExclusive = (resolved.startsWith(VSCODE_SRC_DIST_DIR) || path.dirname(resolved) === path.resolve(DIST_DIR_MAIN, 'service-override')) && !exclusiveModules.has(resolvedWithExtension)
+                  const shouldBeShared = resolvedWithExtension === path.resolve(DIST_DIR_MAIN, 'assets.js') || resolvedWithExtension === path.resolve(DIST_DIR_MAIN, 'lifecycle.js')
+
+                  if (isNotExclusive || shouldBeShared) {
+                    // Those modules will be imported from external monaco-vscode-api
+                    let externalResolved = resolved.startsWith(VSCODE_SRC_DIST_DIR) ? `vscode/vscode/${path.relative(VSCODE_SRC_DIST_DIR, resolved)}` : `vscode/${path.relative(DIST_DIR_MAIN, resolved)}`
+                    if (externalResolved.endsWith('.js')) {
+                      externalResolved = externalResolved.slice(0, -3)
+                    }
+                    return {
+                      id: externalResolved,
+                      external: true
+                    }
+                  }
+
+                  return undefined
+                },
+                load (id) {
+                  if (id === 'entrypoint') {
+                    const codeLines: string[] = []
+                    if ((entrypointInfo.exports ?? []).includes('default')) {
+                      codeLines.push(`export { default } from '${serviceOverrideEntryPoint.slice(0, -3)}'`)
+                    }
+                    if ((entrypointInfo.exports ?? []).some(e => e !== 'default')) {
+                      codeLines.push(`export * from '${serviceOverrideEntryPoint.slice(0, -3)}'`)
+                    }
+                    if ((entrypointInfo.exports ?? []).length === 0) {
+                      codeLines.push(`import '${serviceOverrideEntryPoint.slice(0, -3)}'`)
+                    }
+                    return codeLines.join('\n')
+                  }
+                  if (id === 'worker') {
+                    return `import '${workerEntryPoint}'`
+                  }
+                  if (id.startsWith('vscode/')) {
+                    return (bundle[path.relative('vscode', id)] as rollup.OutputChunk | undefined)?.code
+                  }
+                  return (bundle[path.relative(DIST_DIR_MAIN, id)] as rollup.OutputChunk | undefined)?.code
+                },
+                resolveFileUrl (options) {
+                  let relativePath = options.relativePath
+                  if (!relativePath.startsWith('.')) {
+                    relativePath = `./${options.relativePath}`
+                  }
+                  return `'${relativePath}'`
+                },
+                generateBundle () {
+                  this.emitFile({
+                    fileName: 'package.json',
+                    needsCodeReference: false,
+                    source: JSON.stringify(packageJson, null, 2),
+                    type: 'asset'
+                  })
+                }
+              }]
+          })
+          await groupBundle.write({
+            preserveModules: true,
+            preserveModulesRoot: path.resolve(DIST_DIR, 'main/service-override'),
+            minifyInternalExports: false,
+            assetFileNames: 'assets/[name][extname]',
+            format: 'esm',
+            dir: directory,
+            entryFileNames: '[name].js',
+            chunkFileNames: '[name].js',
+            hoistTransitiveImports: false
+          })
+          await groupBundle.close()
+
+          // remove exclusive files from main bundle to prevent them from being duplicated
+          for (const exclusiveModule of exclusiveModules) {
+            delete bundle[path.relative(DIST_DIR_MAIN, exclusiveModule)]
+          }
         }
       }
-    })
+    }), {
+      name: 'clean-src',
+      async generateBundle () {
+        // Delete intermediate sources because writing to make sur there is no unused files
+        await fsPromise.rm(DIST_DIR_MAIN, {
+          recursive: true
+        })
+      }
+    }
     ]
   }])
 }

rollup/rollup.default-extensions.ts

@@ -113,7 +113,7 @@ export default rollup.defineConfig([
         }
       }),
       metadataPlugin({
-        handle (_, dependencies, entrypoints, options, bundle) {
+        handle (_, dependencies, entrypoints, exclusiveModules, options, bundle) {
           const entrypoint = Object.values(bundle).filter(v => (v as rollup.OutputChunk).isEntry)[0]!.fileName
           const packageJson: PackageJson = {
             name: `@codingame/monaco-vscode-${name}-default-extension`,
@@ -182,7 +182,7 @@ ${extensions.map(name => `  whenReady${pascalCase(name)}()`).join(',\n')}
       }
     },
     metadataPlugin({
-      handle (_, dependencies, entrypoints, options, bundle) {
+      handle (_, dependencies, entrypoints, exclusiveModules, options, bundle) {
         const entrypoint = Object.values(bundle).filter(v => (v as rollup.OutputChunk).isEntry)[0]!.fileName
         const packageJson: PackageJson = {
           name,