From df0306f08d1ebaa9b274c432d1826189fbc0ec14 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Tue, 1 Oct 2024 12:36:16 +0200 Subject: [PATCH] feat!: support readonly fs (#3036) - Closes #2078 BREAKING CHANGE: User files and folders are now symlinked from `/tmp/containerbase/cache` to `/home/ubuntu`. --- src/cli/command/index.ts | 2 + src/cli/command/init-tool.spec.ts | 26 ++++ src/cli/command/init-tool.ts | 47 ++++++ src/cli/install-tool/base-install.service.ts | 21 ++- src/cli/install-tool/index.spec.ts | 16 +- src/cli/install-tool/install-tool.service.ts | 19 ++- src/cli/prepare-tool/base-prepare.service.ts | 9 +- src/cli/prepare-tool/index.spec.ts | 30 +++- src/cli/prepare-tool/index.ts | 10 +- .../prepare-legacy-tools.service.ts | 13 +- src/cli/prepare-tool/prepare-tool.service.ts | 142 +++++++++++++----- src/cli/services/env.service.ts | 4 + src/cli/services/http.service.ts | 2 +- src/cli/services/path.service.spec.ts | 50 ++++-- src/cli/services/path.service.ts | 134 ++++++++++++++--- src/cli/services/version.service.ts | 6 +- src/cli/tools/dart/index.ts | 15 +- src/cli/tools/dart/utils.ts | 33 ++-- src/cli/tools/docker.ts | 2 +- src/cli/tools/dotnet.ts | 33 ++-- src/cli/tools/flutter.ts | 40 +++-- src/cli/tools/index.ts | 2 + src/cli/tools/java/index.ts | 49 +++--- src/cli/tools/java/utils.ts | 16 +- src/cli/tools/node/index.ts | 22 ++- src/cli/tools/node/utils.ts | 23 +-- src/cli/tools/php/index.ts | 2 +- src/cli/tools/python/conan.ts | 17 ++- src/cli/utils/common.spec.ts | 15 +- src/cli/utils/common.ts | 20 ++- src/usr/local/containerbase/bin/init-tool.sh | 23 +++ .../local/containerbase/bin/install-tool.sh | 4 - .../local/containerbase/bin/prepare-tool.sh | 20 +-- .../local/containerbase/tools/v2/erlang.sh | 13 +- .../local/containerbase/tools/v2/golang.sh | 29 ++-- src/usr/local/containerbase/tools/v2/nix.sh | 2 +- .../containerbase/tools/v2/powershell.sh | 12 +- .../local/containerbase/tools/v2/python.sh | 13 +- src/usr/local/containerbase/tools/v2/ruby.sh | 71 +++++---- src/usr/local/containerbase/tools/v2/rust.sh | 33 ++-- src/usr/local/containerbase/tools/v2/swift.sh | 14 +- src/usr/local/containerbase/util.sh | 2 + .../local/containerbase/utils/constants.sh | 3 + .../local/containerbase/utils/filesystem.sh | 112 ++++++++++---- src/usr/local/containerbase/utils/init.sh | 17 +++ src/usr/local/containerbase/utils/install.sh | 13 +- src/usr/local/containerbase/utils/linking.sh | 7 + src/usr/local/containerbase/utils/prepare.sh | 21 +++ .../local/containerbase/utils/v2/defaults.sh | 7 +- test/bash/filesystem.bats | 10 +- test/bash/util.sh | 2 + test/bash/v2/defaults.bats | 1 - test/java/Dockerfile | 17 +-- test/node/Dockerfile | 53 ++++--- test/rust/Dockerfile | 5 + 55 files changed, 949 insertions(+), 375 deletions(-) create mode 100644 src/cli/command/init-tool.spec.ts create mode 100644 src/cli/command/init-tool.ts create mode 100755 src/usr/local/containerbase/bin/init-tool.sh create mode 100644 src/usr/local/containerbase/utils/init.sh diff --git a/src/cli/command/index.ts b/src/cli/command/index.ts index 246d510bf..1b391159c 100644 --- a/src/cli/command/index.ts +++ b/src/cli/command/index.ts @@ -2,6 +2,7 @@ import type { Cli } from 'clipanion'; import type { CliMode } from '../utils'; import { logger } from '../utils/logger'; import { DownloadFileCommand } from './download-file'; +import { InitToolCommand } from './init-tool'; import { InstallGemCommand, InstallGemShortCommand } from './install-gem'; import { InstallNpmCommand, InstallNpmShortCommand } from './install-npm'; import { InstallPipCommand, InstallPipShortCommand } from './install-pip'; @@ -40,4 +41,5 @@ export function prepareCommands(cli: Cli, mode: CliMode | null): void { cli.register(InstallPipCommand); cli.register(InstallToolCommand); cli.register(PrepareToolCommand); + cli.register(InitToolCommand); } diff --git a/src/cli/command/init-tool.spec.ts b/src/cli/command/init-tool.spec.ts new file mode 100644 index 000000000..c33ddb0e8 --- /dev/null +++ b/src/cli/command/init-tool.spec.ts @@ -0,0 +1,26 @@ +import { Cli } from 'clipanion'; +import { describe, expect, test, vi } from 'vitest'; +import { prepareCommands } from '.'; + +const mocks = vi.hoisted(() => ({ + installTool: vi.fn(), + prepareTools: vi.fn(), + initializeTools: vi.fn(), +})); + +vi.mock('../install-tool', () => mocks); +vi.mock('../prepare-tool', () => mocks); + +describe('index', () => { + test('init-tool', async () => { + const cli = new Cli({ binaryName: 'cli' }); + prepareCommands(cli, null); + + expect(await cli.run(['init', 'tool', 'node'])).toBe(0); + expect(mocks.initializeTools).toHaveBeenCalledOnce(); + expect(mocks.initializeTools).toHaveBeenCalledWith(['node'], false); + + mocks.initializeTools.mockRejectedValueOnce(new Error('test')); + expect(await cli.run(['init', 'tool', 'node'])).toBe(1); + }); +}); diff --git a/src/cli/command/init-tool.ts b/src/cli/command/init-tool.ts new file mode 100644 index 000000000..cca00c815 --- /dev/null +++ b/src/cli/command/init-tool.ts @@ -0,0 +1,47 @@ +import { Command, Option } from 'clipanion'; +import prettyMilliseconds from 'pretty-ms'; +import { initializeTools } from '../prepare-tool'; +import { logger } from '../utils'; + +export class InitToolCommand extends Command { + static override paths = [['init', 'tool']]; + + static override usage = Command.Usage({ + description: + 'Initialize a tool into the container. This creates missing files and directories.', + examples: [ + ['Initialize node', '$0 init tool node'], + ['Initialize all tools', '$0 init tool all'], + ], + }); + + tools = Option.Rest({ required: 1 }); + + dryRun = Option.Boolean('-d,--dry-run', false); + + async execute(): Promise { + const start = Date.now(); + let error = false; + logger.info(`Initializing tools ${this.tools.join(', ')}...`); + try { + return await initializeTools(this.tools, this.dryRun); + } catch (err) { + error = true; + logger.debug(err); + if (err instanceof Error) { + logger.fatal(err.message); + } + return 1; + } finally { + if (error) { + logger.fatal( + `Initialize tools ${this.tools.join(', ')} failed in ${prettyMilliseconds(Date.now() - start)}.`, + ); + } else { + logger.info( + `Initialize tools ${this.tools.join(', ')} succeded in ${prettyMilliseconds(Date.now() - start)}.`, + ); + } + } + } +} diff --git a/src/cli/install-tool/base-install.service.ts b/src/cli/install-tool/base-install.service.ts index f1dfe90d8..1e091182a 100644 --- a/src/cli/install-tool/base-install.service.ts +++ b/src/cli/install-tool/base-install.service.ts @@ -3,8 +3,8 @@ import { join } from 'node:path'; import { codeBlock } from 'common-tags'; import { injectable } from 'inversify'; import type { EnvService, PathService } from '../services'; -import { NoPrepareTools } from '../tools'; -import { isValid } from '../utils'; +import { NoInitTools, NoPrepareTools } from '../tools'; +import { isValid, tool2path } from '../utils'; export interface ShellWrapperConfig { name?: string; @@ -35,12 +35,20 @@ export abstract class BaseInstallService { return !!(await this.pathSvc.findVersionedToolPath(this.name, version)); } + async isInitialized(): Promise { + return await this.pathSvc.isInitialized(this.name); + } + async isPrepared(): Promise { - return null !== (await this.pathSvc.findToolPath(this.name)); + return await this.pathSvc.isPrepared(this.name); } abstract link(version: string): Promise; + needsInitialize(): boolean { + return !NoInitTools.includes(this.name); + } + needsPrepare(): boolean { return !NoPrepareTools.includes(this.name); } @@ -70,7 +78,7 @@ export abstract class BaseInstallService { }: ShellWrapperConfig): Promise { const tgt = join(this.pathSvc.binDir, name ?? this.name); - const envs = [...(extraToolEnvs ?? []), this.name]; + const envs = [...(extraToolEnvs ?? []), this.name].map(tool2path); let content = codeBlock` #!/bin/bash @@ -78,6 +86,11 @@ export abstract class BaseInstallService { . ${this.pathSvc.envFile} fi + if [[ ! -f "${this.pathSvc.toolInitPath(this.name)}" ]]; then + # set logging to only warn and above to not interfere with tool output + CONTAINERBASE_LOG_LEVEL=warn containerbase-cli init tool "${this.name}" + fi + # load tool envs for n in ${envs.join(' ')}; do if [[ -f "${this.pathSvc.toolsPath}/\${n}/env.sh" ]]; then diff --git a/src/cli/install-tool/index.spec.ts b/src/cli/install-tool/index.spec.ts index 8edba556f..7b6b4a002 100644 --- a/src/cli/install-tool/index.spec.ts +++ b/src/cli/install-tool/index.spec.ts @@ -1,4 +1,6 @@ -import { describe, expect, test, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import { beforeAll, describe, expect, test, vi } from 'vitest'; +import { rootPath } from '../../../test/path'; import { installTool, resolveVersion } from '.'; vi.mock('del'); @@ -7,6 +9,18 @@ vi.mock('../tools/bun'); vi.mock('../tools/php/composer'); describe('index', () => { + beforeAll(async () => { + for (const p of [ + 'var/lib/containerbase/tool.prep.d', + 'tmp/containerbase/tool.init.d', + ]) { + const prepDir = rootPath(p); + await fs.mkdir(prepDir, { + recursive: true, + }); + } + }); + test('installTool', async () => { expect(await installTool('bun', '1.0.0')).toBeUndefined(); expect(await installTool('dummy', '1.0.0')).toBeUndefined(); diff --git a/src/cli/install-tool/install-tool.service.ts b/src/cli/install-tool/install-tool.service.ts index b257adbc7..4df478e10 100644 --- a/src/cli/install-tool/install-tool.service.ts +++ b/src/cli/install-tool/install-tool.service.ts @@ -1,6 +1,6 @@ import { deleteAsync } from 'del'; import { inject, injectable, multiInject, optional } from 'inversify'; -import { prepareTools } from '../prepare-tool'; +import { initializeTools, prepareTools } from '../prepare-tool'; import { EnvService, PathService, VersionService } from '../services'; import { cleanAptFiles, cleanTmpFiles, isDockerBuild, logger } from '../utils'; import type { BaseInstallService } from './base-install.service'; @@ -31,6 +31,8 @@ export class InstallToolService { 'supported tools', ); + await this.pathSvc.ensureBasePaths(); + try { const toolSvc = this.toolSvcs.find((t) => t.name === tool); if (toolSvc) { @@ -48,6 +50,14 @@ export class InstallToolService { } } + if (toolSvc.needsInitialize() && !(await toolSvc.isInitialized())) { + logger.debug({ tool }, 'tool not initialized'); + const res = await initializeTools([tool], dryRun); + if (res) { + return res; + } + } + logger.debug({ tool }, 'validate tool'); if (!(await toolSvc.validate(version))) { logger.fatal({ tool, version }, 'tool version not supported'); @@ -83,7 +93,7 @@ export class InstallToolService { if (await isDockerBuild()) { logger.debug('cleaning tmp files'); - await cleanTmpFiles(this.pathSvc.tmpDir, dryRun); + await cleanTmpFiles(this.envSvc.tmpDir, dryRun); if (this.envSvc.isRoot) { logger.debug('cleaning root caches'); @@ -96,9 +106,8 @@ export class InstallToolService { logger.debug('cleaning user caches'); await deleteAsync( [ - `${this.envSvc.userHome}/.cache`, - `${this.envSvc.userHome}/.local/share/virtualenv`, - `${this.pathSvc.cachePath}/**`, + `${this.pathSvc.cachePath}/.cache/**`, + `${this.pathSvc.cachePath}/.local/share/virtualenv/**`, ], { force: true, diff --git a/src/cli/prepare-tool/base-prepare.service.ts b/src/cli/prepare-tool/base-prepare.service.ts index 6c1b0f3c6..b691eb5f3 100644 --- a/src/cli/prepare-tool/base-prepare.service.ts +++ b/src/cli/prepare-tool/base-prepare.service.ts @@ -10,10 +10,11 @@ export abstract class BasePrepareService { @inject(EnvService) protected readonly envSvc: EnvService, ) {} - abstract execute(): Promise | void; - - async isPrepared(): Promise { - return null !== (await this.pathSvc.findToolPath(this.name)); + prepare(): Promise | void { + // noting to do; + } + initialize(): Promise | void { + // noting to do; } toString(): string { diff --git a/src/cli/prepare-tool/index.spec.ts b/src/cli/prepare-tool/index.spec.ts index 6b9f22af8..371f55549 100644 --- a/src/cli/prepare-tool/index.spec.ts +++ b/src/cli/prepare-tool/index.spec.ts @@ -1,5 +1,7 @@ -import { describe, expect, test, vi } from 'vitest'; -import { prepareTools } from '.'; +import fs from 'node:fs/promises'; +import { beforeAll, describe, expect, test, vi } from 'vitest'; +import { rootPath } from '../../../test/path'; +import { initializeTools, prepareTools } from '.'; vi.mock('del'); vi.mock('execa'); @@ -12,7 +14,31 @@ vi.mock('node:process', async (importOriginal) => ({ })); describe('index', () => { + beforeAll(async () => { + for (const p of [ + 'var/lib/containerbase/tool.prep.d', + 'tmp/containerbase/tool.init.d', + 'usr/local/containerbase/tools/v2', + ]) { + const prepDir = rootPath(p); + await fs.mkdir(prepDir, { + recursive: true, + }); + } + + await fs.writeFile( + rootPath('usr/local/containerbase/tools/v2/dummy.sh'), + '', + ); + }); + test('prepareTools', async () => { expect(await prepareTools(['bun', 'dummy'])).toBeUndefined(); + expect(await prepareTools(['not-exist'])).toBe(1); + }); + + test('initializeTools', async () => { + expect(await initializeTools(['bun', 'dummy'])).toBeUndefined(); + expect(await initializeTools(['not-exist'])).toBeUndefined(); }); }); diff --git a/src/cli/prepare-tool/index.ts b/src/cli/prepare-tool/index.ts index 3d9bdcdbc..34025d1c0 100644 --- a/src/cli/prepare-tool/index.ts +++ b/src/cli/prepare-tool/index.ts @@ -46,5 +46,13 @@ export function prepareTools( dryRun = false, ): Promise { const container = prepareContainer(); - return container.get(PrepareToolService).execute(tools, dryRun); + return container.get(PrepareToolService).prepare(tools, dryRun); +} + +export function initializeTools( + tools: string[], + dryRun = false, +): Promise { + const container = prepareContainer(); + return container.get(PrepareToolService).initialize(tools, dryRun); } diff --git a/src/cli/prepare-tool/prepare-legacy-tools.service.ts b/src/cli/prepare-tool/prepare-legacy-tools.service.ts index 6691a62c8..3f17630a7 100644 --- a/src/cli/prepare-tool/prepare-legacy-tools.service.ts +++ b/src/cli/prepare-tool/prepare-legacy-tools.service.ts @@ -4,9 +4,16 @@ import { logger } from '../utils'; @injectable() export class PrepareLegacyToolsService { - async execute(tools: string[]): Promise { - logger.debug(`Preparing legacy tools ${tools.join(', ')} ...`); - await execa('/usr/local/containerbase/bin/prepare-tool.sh', tools, { + async prepare(tool: string): Promise { + logger.debug(`Preparing legacy tool ${tool} ...`); + await execa('/usr/local/containerbase/bin/prepare-tool.sh', [tool], { + stdio: ['inherit', 'inherit', 1], + }); + } + + async initialize(tool: string): Promise { + logger.debug(`Initializing legacy tool ${tool} ...`); + await execa('/usr/local/containerbase/bin/init-tool.sh', [tool], { stdio: ['inherit', 'inherit', 1], }); } diff --git a/src/cli/prepare-tool/prepare-tool.service.ts b/src/cli/prepare-tool/prepare-tool.service.ts index 6406000da..a1a6dd759 100644 --- a/src/cli/prepare-tool/prepare-tool.service.ts +++ b/src/cli/prepare-tool/prepare-tool.service.ts @@ -1,7 +1,7 @@ import { deleteAsync } from 'del'; import { inject, injectable, multiInject, optional } from 'inversify'; import { EnvService, PathService } from '../services'; -import { NoPrepareTools } from '../tools'; +import { NoInitTools, NoPrepareTools } from '../tools'; import { cleanAptFiles, cleanTmpFiles, logger } from '../utils'; import type { BasePrepareService } from './base-prepare.service'; import { PrepareLegacyToolsService } from './prepare-legacy-tools.service'; @@ -20,11 +20,12 @@ export class PrepareToolService { @inject(EnvService) private envSvc: EnvService, ) {} - async execute(tools: string[], dryRun = false): Promise { - logger.trace( - { tools: this.toolSvcs.map((t) => t.name) }, - 'supported tools', - ); + async prepare(tools: string[], dryRun = false): Promise { + const supportedTools = [ + ...this.toolSvcs.map((t) => t.name), + ...(await this.pathSvc.findLegacyTools()), + ].sort(); + logger.trace({ tools: supportedTools }, 'supported tools'); if (dryRun) { logger.info(`Dry run: preparing tools ${tools.join(', ')} ...`); return; @@ -35,47 +36,25 @@ export class PrepareToolService { } try { if (tools.length === 1 && tools[0] === 'all') { - for (const tool of this.toolSvcs) { - if (this.envSvc.isToolIgnored(tool.name)) { - logger.info({ tool }, 'tool ignored'); - continue; + for (const tool of supportedTools) { + const res = await this._prepareTool(tool, dryRun); + if (res) { + return res; } - if (await tool.isPrepared()) { - logger.debug({ tool: tool.name }, 'tool already prepared'); - continue; - } - logger.debug({ tool: tool.name }, 'preparing tool'); - await tool.execute(); - await this.pathSvc.ensureToolPath(tool.name); + await this.pathSvc.setPrepared(tool); } - await this.legacySvc.execute(tools); } else { for (const tool of tools) { - if (this.envSvc.isToolIgnored(tool)) { - logger.info({ tool }, 'tool ignored'); - continue; - } - if (NoPrepareTools.includes(tool)) { - logger.info({ tool }, 'tool does not need to be prepared'); - continue; - } - const toolSvc = this.toolSvcs.find((t) => t.name === tool); - if (toolSvc) { - if (await toolSvc.isPrepared()) { - logger.debug({ tool }, 'tool already prepared'); - continue; - } - logger.debug({ tool }, 'preparing tool'); - await toolSvc.execute(); - await this.pathSvc.ensureToolPath(tool); - } else { - await this.legacySvc.execute([tool]); + const res = await this._prepareTool(tool, dryRun); + if (res) { + return res; } + await this.pathSvc.setPrepared(tool); } } } finally { await cleanAptFiles(dryRun); - await cleanTmpFiles(this.pathSvc.tmpDir, dryRun); + await cleanTmpFiles(this.envSvc.tmpDir, dryRun); await deleteAsync(['/root/.cache', '/root/.local/share/virtualenv'], { force: true, dryRun, @@ -83,4 +62,91 @@ export class PrepareToolService { }); } } + + async initialize(tools: string[], dryRun = false): Promise { + const supportedTools = [ + ...this.toolSvcs.map((t) => t.name), + ...(await this.pathSvc.findLegacyTools()), + ].sort(); + logger.trace({ tools: supportedTools }, 'supported tools'); + if (dryRun) { + logger.info(`Dry run: initializing tools ${tools.join(', ')} ...`); + return; + } + + await this.pathSvc.ensureBasePaths(); + + if (tools.length === 1 && tools[0] === 'all') { + for (const tool of supportedTools) { + const res = await this._initTool(tool, dryRun); + if (res) { + return res; + } + await this.pathSvc.setInitialized(tool); + } + } else { + for (const tool of tools) { + const res = await this._initTool(tool, dryRun); + if (res) { + return res; + } + await this.pathSvc.setInitialized(tool); + } + } + } + + private async _initTool( + tool: string, + _dryRun: boolean, + ): Promise { + if (this.envSvc.isToolIgnored(tool)) { + logger.info({ tool }, 'tool ignored'); + return; + } + if (NoInitTools.includes(tool)) { + logger.info({ tool }, 'tool does not need to be initialized'); + return; + } + if (await this.pathSvc.isInitialized(tool)) { + logger.debug({ tool }, 'tool already initialized'); + return; + } + const toolSvc = this.toolSvcs.find((t) => t.name === tool); + if (toolSvc) { + logger.debug({ tool }, 'initialize tool'); + await toolSvc.initialize(); + await this.pathSvc.ensureToolPath(tool); + } else if (await this.pathSvc.isLegacyTool(tool)) { + await this.legacySvc.initialize(tool); + } // ignore else + } + + private async _prepareTool( + tool: string, + _dryRun: boolean, + ): Promise { + if (this.envSvc.isToolIgnored(tool)) { + logger.info({ tool }, 'tool ignored'); + return; + } + if (NoPrepareTools.includes(tool)) { + logger.info({ tool }, 'tool does not need to be prepared'); + return; + } + if (await this.pathSvc.isPrepared(tool)) { + logger.debug({ tool }, 'tool already prepared'); + return; + } + const toolSvc = this.toolSvcs.find((t) => t.name === tool); + if (toolSvc) { + logger.debug({ tool }, 'preparing tool'); + await toolSvc.prepare(); + await this.pathSvc.ensureToolPath(tool); + } else if (await this.pathSvc.isLegacyTool(tool)) { + await this.legacySvc.prepare(tool); + } else { + logger.error({ tool }, 'tool not found'); + return 1; + } + } } diff --git a/src/cli/services/env.service.ts b/src/cli/services/env.service.ts index 471608f62..250e55c43 100644 --- a/src/cli/services/env.service.ts +++ b/src/cli/services/env.service.ts @@ -68,6 +68,10 @@ export class EnvService { return join(this.rootDir, 'root'); } + get tmpDir(): string { + return join(this.rootDir, 'tmp'); + } + get userHome(): string { return env.USER_HOME ?? join(this.rootDir, 'home', this.userName); } diff --git a/src/cli/services/http.service.ts b/src/cli/services/http.service.ts index df209d4ae..f00989ca4 100644 --- a/src/cli/services/http.service.ts +++ b/src/cli/services/http.service.ts @@ -57,7 +57,7 @@ export class HttpService { const urlChecksum = hash(url, 'sha256'); - const cacheDir = this.envSvc.cacheDir ?? this.pathSvc.tmpDir; + const cacheDir = this.envSvc.cacheDir ?? this.envSvc.tmpDir; const cachePath = join(cacheDir, urlChecksum); // TODO: validate name const file = fileName ?? new URL(url).pathname.split('/').pop()!; diff --git a/src/cli/services/path.service.spec.ts b/src/cli/services/path.service.spec.ts index c8e570b76..6013744f2 100644 --- a/src/cli/services/path.service.spec.ts +++ b/src/cli/services/path.service.spec.ts @@ -17,11 +17,17 @@ describe('path.service', () => { env.PATH = path; delete env.NODE_VERSION; await deleteAsync('**', { force: true, dot: true, cwd: rootPath() }); + await mkdir(rootPath('var/lib/containerbase/tool.prep.d'), { + recursive: true, + }); + await mkdir(rootPath('tmp/containerbase/tool.init.d'), { + recursive: true, + }); }); test('cachePath', () => { expect(child.get(PathService).cachePath).toBe( - rootPath('opt/containerbase/cache'), + rootPath('tmp/containerbase/cache'), ); }); @@ -30,13 +36,7 @@ describe('path.service', () => { }); test('tmpDir', () => { - expect(child.get(PathService).tmpDir).toBe(rootPath('tmp')); - }); - - test('homePath', () => { - expect(child.get(PathService).homePath).toBe( - rootPath('opt/containerbase/home'), - ); + expect(child.get(PathService).tmpDir).toBe(rootPath('tmp/containerbase')); }); test('sslPath', () => { @@ -65,6 +65,15 @@ describe('path.service', () => { expect(await child.get(PathService).toolEnvExists('node')).toBe(false); }); + test('ensureBasePaths', async () => { + await child.get(PathService).ensureBasePaths(); + expect(await pathExists(rootPath('opt/containerbase'), 'dir')).toBe(true); + expect(await pathExists(rootPath('var/lib/containerbase'), 'dir')).toBe( + true, + ); + expect(await pathExists(rootPath('tmp/containerbase'), 'dir')).toBe(true); + }); + test('exportToolEnvContent', async () => { const pathSvc = child.get(PathService); @@ -169,7 +178,7 @@ describe('path.service', () => { rootPath('opt/containerbase/tools/node'), ); expect( - await pathExists(rootPath('opt/containerbase/tools/node'), true), + await pathExists(rootPath('opt/containerbase/tools/node'), 'dir'), ).toBe(true); }); @@ -180,11 +189,32 @@ describe('path.service', () => { }); test('createDir', async () => { - const dir = rootPath('env123'); + const dir = rootPath('env123/sub'); expect(await child.get(PathService).createDir(dir)).toBeUndefined(); const s = await stat(dir); expect(s.mode & fileRights).toBe(platform() === 'win32' ? 0 : 0o775); + expect(await child.get(PathService).createDir(dir)).toBeUndefined(); + }); + + test('toolInit', async () => { + const pathSvc = child.get(PathService); + expect(pathSvc.toolInitPath('node')).toBe( + rootPath('tmp/containerbase/tool.init.d/node'), + ); + expect(await pathSvc.isInitialized('node')).toBe(false); + await pathSvc.setInitialized('node'); + expect(await pathSvc.isInitialized('node')).toBe(true); + }); + + test('toolPrepare', async () => { + const pathSvc = child.get(PathService); + expect(pathSvc.toolPreparePath('node')).toBe( + rootPath('var/lib/containerbase/tool.prep.d/node'), + ); + expect(await pathSvc.isPrepared('node')).toBe(false); + await pathSvc.setPrepared('node'); + expect(await pathSvc.isPrepared('node')).toBe(true); }); test('writeFile', async () => { diff --git a/src/cli/services/path.service.ts b/src/cli/services/path.service.ts index 53227afc7..08fa472a4 100644 --- a/src/cli/services/path.service.ts +++ b/src/cli/services/path.service.ts @@ -2,7 +2,7 @@ import fs from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { env } from 'node:process'; import { inject, injectable } from 'inversify'; -import { fileRights, logger, pathExists } from '../utils'; +import { fileRights, logger, pathExists, tool2path } from '../utils'; import { EnvService } from './env.service'; export interface FileOwnerConfig { @@ -15,38 +15,82 @@ export interface FileOwnerConfig { @injectable() export class PathService { + /** + * Path to `/tmp/containerbase/tool.init.d`. + */ + private get _toolInitPath(): string { + return join(this.tmpDir, 'tool.init.d'); + } + /** + * Path to `/var/lib/containerbase/tool.prep.d`. + */ + private get _toolPrepPath(): string { + return join(this.varPath, 'tool.prep.d'); + } + + /** + * Path to `/opt/containerbase/bin`. + */ get binDir(): string { return join(this.installDir, 'bin'); } + /** + * Path to `/tmp/containerbase/cache`. + */ get cachePath(): string { - return join(this.installDir, 'cache'); + return join(this.tmpDir, 'cache'); } get envFile(): string { return join(this.envSvc.rootDir, 'usr/local/etc/env'); } - get homePath(): string { - return join(this.installDir, 'home'); - } - + /** + * Path to `/opt/containerbase`. + */ get installDir(): string { return join(this.envSvc.rootDir, 'opt/containerbase'); } + /** + * Path to `/opt/containerbase/ssl`. + */ get sslPath(): string { return join(this.installDir, 'ssl'); } + /** + * Path to `/tmp/containerbase`. + */ get tmpDir(): string { - return join(this.envSvc.rootDir, 'tmp'); + return join(this.envSvc.tmpDir, 'containerbase'); } + /** + * Path to `/opt/containerbase/tools`. + */ get toolsPath(): string { return join(this.installDir, 'tools'); } + /** + * Path to `/usr/local/containerbase`. + */ + get usrPath(): string { + return join(this.envSvc.rootDir, 'usr/local/containerbase'); + } + + /** + * Path to `/var/lib/containerbase`. + */ + get varPath(): string { + return join(this.envSvc.rootDir, 'var/lib/containerbase'); + } + + /** + * Path to `/opt/containerbase/versions`. + */ get versionPath(): string { return join(this.installDir, 'versions'); } @@ -54,27 +98,23 @@ export class PathService { constructor(@inject(EnvService) private envSvc: EnvService) {} async createDir(path: string, mode = 0o775): Promise { - if (await pathExists(path, true)) { + if (await pathExists(path)) { return; } const parent = dirname(path); - if (!(await pathExists(parent, true))) { + if (!(await pathExists(parent))) { await this.createDir(parent, 0o775); } + logger.debug({ path }, 'creating dir'); await fs.mkdir(path); await this.setOwner({ path, mode }); } - async createToolPath(tool: string): Promise { const toolPath = this.toolPath(tool); await this.createDir(toolPath); return toolPath; } - async ensureToolPath(tool: string): Promise { - return (await this.findToolPath(tool)) ?? (await this.createToolPath(tool)); - } - async createVersionedToolPath( tool: string, version: string, @@ -84,10 +124,26 @@ export class PathService { return toolPath; } + async ensureBasePaths(): Promise { + if (!(await pathExists(this.varPath, 'dir'))) { + throw new Error('System not initialized for containerbase'); + } + await this.createDir(this._toolPrepPath); + await this.createDir(this.toolsPath); + await this.createDir(this.versionPath); + await this.createDir(this.binDir); + await this.createDir(this.sslPath); + await this.createDir(this._toolInitPath); + } + + async ensureToolPath(tool: string): Promise { + return (await this.findToolPath(tool)) ?? (await this.createToolPath(tool)); + } + async findToolPath(tool: string): Promise { const toolPath = this.toolPath(tool); - if (await pathExists(toolPath, true)) { + if (await pathExists(toolPath, 'dir')) { return toolPath; } return null; @@ -99,18 +155,57 @@ export class PathService { ): Promise { const versionedToolPath = this.versionedToolPath(tool, version); - if (await pathExists(versionedToolPath, true)) { + if (await pathExists(versionedToolPath, 'dir')) { return versionedToolPath; } return null; } + async findLegacyTools(): Promise { + const tools = await fs.readdir(join(this.usrPath, 'tools/v2')); + return tools + .filter((t) => t.endsWith('.sh')) + .map((t) => t.substring(0, t.length - 3)); + } + async fileExists(filePath: string): Promise { - return await pathExists(filePath); + return await pathExists(filePath, 'file'); + } + + async isInitialized(tool: string): Promise { + return await this.fileExists(this.toolInitPath(tool)); + } + + async isPrepared(tool: string): Promise { + return await this.fileExists(this.toolPreparePath(tool)); + } + + async isLegacyTool(tool: string, v1 = false): Promise { + let exists = await pathExists(join(this.usrPath, 'tools/v2', `${tool}.sh`)); + if (!exists && v1) { + exists = await pathExists(join(this.usrPath, 'tools', `${tool}.sh`)); + } + return exists; + } + + async setInitialized(tool: string): Promise { + await fs.writeFile(this.toolInitPath(tool), ''); + } + + async setPrepared(tool: string): Promise { + await fs.writeFile(this.toolPreparePath(tool), ''); + } + + toolInitPath(tool: string): string { + return join(this._toolInitPath, tool2path(tool)); } toolPath(tool: string): string { - return join(this.toolsPath, tool); + return join(this.toolsPath, tool2path(tool)); + } + + toolPreparePath(tool: string): string { + return join(this._toolPrepPath, tool2path(tool)); } versionedToolPath(tool: string, version: string): string { @@ -191,8 +286,7 @@ export class PathService { content += 'fi\n'; } - await fs.appendFile(file, content); - await this.setOwner({ path: file, mode: 0o664 }); + await this.writeFile(file, content); } async exportToolPath( diff --git a/src/cli/services/version.service.ts b/src/cli/services/version.service.ts index 211532b65..f2d9ca480 100644 --- a/src/cli/services/version.service.ts +++ b/src/cli/services/version.service.ts @@ -1,7 +1,7 @@ import { chmod, readFile, stat, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { inject, injectable } from 'inversify'; -import { fileRights, logger } from '../utils'; +import { fileRights, logger, tool2path } from '../utils'; import { PathService } from './path.service'; @injectable() @@ -9,7 +9,7 @@ export class VersionService { constructor(@inject(PathService) private pathSvc: PathService) {} async find(tool: string): Promise { - const path = join(this.pathSvc.versionPath, tool); + const path = join(this.pathSvc.versionPath, tool2path(tool)); try { return (await readFile(path, { encoding: 'utf8' })).trim() || null; } catch (err) { @@ -24,7 +24,7 @@ export class VersionService { } async update(tool: string, version: string): Promise { - const path = join(this.pathSvc.versionPath, tool); + const path = join(this.pathSvc.versionPath, tool2path(tool)); try { await writeFile(path, version, { encoding: 'utf8' }); const s = await stat(path); diff --git a/src/cli/tools/dart/index.ts b/src/cli/tools/dart/index.ts index c0f84f342..35f8afaf3 100644 --- a/src/cli/tools/dart/index.ts +++ b/src/cli/tools/dart/index.ts @@ -11,7 +11,12 @@ import { PathService, } from '../../services'; import { parse } from '../../utils'; -import { prepareDartHome, preparePubCache } from './utils'; +import { + initDartHome, + initPubCache, + prepareDartHome, + preparePubCache, +} from './utils'; // Dart SDK sample urls // https://storage.googleapis.com/dart-archive/channels/stable/release/1.11.0/sdk/dartsdk-linux-x64-release.zip @@ -24,10 +29,16 @@ import { prepareDartHome, preparePubCache } from './utils'; export class DartPrepareService extends BasePrepareService { readonly name = 'dart'; - async execute(): Promise { + override async prepare(): Promise { + await this.initialize(); await prepareDartHome(this.envSvc, this.pathSvc); await preparePubCache(this.envSvc, this.pathSvc); } + + override async initialize(): Promise { + await initDartHome(this.pathSvc); + await initPubCache(this.pathSvc); + } } @injectable() diff --git a/src/cli/tools/dart/utils.ts b/src/cli/tools/dart/utils.ts index 01ef48820..65f32411a 100644 --- a/src/cli/tools/dart/utils.ts +++ b/src/cli/tools/dart/utils.ts @@ -7,20 +7,31 @@ export async function prepareDartHome( envSvc: EnvService, pathSvc: PathService, ): Promise { + const dart = join(envSvc.userHome, '.dart'); + if (!(await pathExists(dart))) { + await fs.symlink(join(pathSvc.cachePath, '.dart'), dart); + await fs.symlink( + join(pathSvc.cachePath, '.dart-tool'), + join(envSvc.userHome, '.dart-tool'), + ); + } + // for root const rootDart = join(envSvc.rootDir, 'root', '.dart'); - if (!(await pathExists(rootDart, true))) { + if (!(await pathExists(rootDart))) { await fs.mkdir(rootDart); await fs.writeFile( join(rootDart, 'dartdev.json'), '{ "firstRun": false, "enabled": false }', ); } +} +export async function initDartHome(pathSvc: PathService): Promise { // for user - const dart = join(pathSvc.homePath, '.dart'); - if (await pathExists(dart, true)) { - const dartTool = join(pathSvc.homePath, '.dart-tool'); + const dart = join(pathSvc.cachePath, '.dart'); + if (!(await pathExists(dart))) { + const dartTool = join(pathSvc.cachePath, '.dart-tool'); await pathSvc.createDir(dart); await pathSvc.createDir(dartTool); @@ -30,9 +41,6 @@ export async function prepareDartHome( const dartToolTelemetry = join(dartTool, 'dart-flutter-telemetry.config'); await pathSvc.writeFile(dartToolTelemetry, 'reporting=0\n'); - - await fs.symlink(dart, join(envSvc.userHome, '.dart')); - await fs.symlink(dartTool, join(envSvc.userHome, '.dart-tool')); } } @@ -40,9 +48,12 @@ export async function preparePubCache( envSvc: EnvService, pathSvc: PathService, ): Promise { - const pubCache = join(pathSvc.homePath, '.pub-cache'); - if (!(await pathExists(pubCache, true))) { - await pathSvc.createDir(pubCache); - await fs.symlink(pubCache, join(envSvc.userHome, '.pub-cache')); + const pubCache = join(envSvc.userHome, '.pub-cache'); + if (!(await pathExists(pubCache))) { + await fs.symlink(join(pathSvc.cachePath, '.pub-cache'), pubCache); } } + +export async function initPubCache(pathSvc: PathService): Promise { + await pathSvc.createDir(join(pathSvc.cachePath, '.pub-cache')); +} diff --git a/src/cli/tools/docker.ts b/src/cli/tools/docker.ts index 2721a7ad8..041e612f8 100644 --- a/src/cli/tools/docker.ts +++ b/src/cli/tools/docker.ts @@ -15,7 +15,7 @@ import { export class DockerPrepareService extends BasePrepareService { readonly name = 'docker'; - async execute(): Promise { + override async prepare(): Promise { await execa('groupadd', ['-g', '999', 'docker']); await execa('usermod', ['-aG', 'docker', this.envSvc.userName]); } diff --git a/src/cli/tools/dotnet.ts b/src/cli/tools/dotnet.ts index 59c56c329..4b4ed9e4d 100644 --- a/src/cli/tools/dotnet.ts +++ b/src/cli/tools/dotnet.ts @@ -11,7 +11,7 @@ import { HttpService, PathService, } from '../services'; -import { getDistro, parse } from '../utils'; +import { getDistro, parse, pathExists } from '../utils'; @injectable() export class DotnetPrepareService extends BasePrepareService { @@ -25,7 +25,7 @@ export class DotnetPrepareService extends BasePrepareService { super(pathSvc, envSvc); } - async execute(): Promise { + override async prepare(): Promise { const distro = await getDistro(); switch (distro.versionCode) { @@ -64,17 +64,26 @@ export class DotnetPrepareService extends BasePrepareService { break; } - await this.pathSvc.exportToolEnv(this.name, { - DOTNET_ROOT: this.pathSvc.toolPath(this.name), - DOTNET_CLI_TELEMETRY_OPTOUT: '1', - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: '1', - }); + await this.initialize(); + await fs.symlink( + join(this.pathSvc.cachePath, '.nuget'), + join(this.envSvc.userHome, '.nuget'), + ); + } + + override async initialize(): Promise { + if (!(await this.pathSvc.toolEnvExists(this.name))) { + await this.pathSvc.exportToolEnv(this.name, { + DOTNET_ROOT: this.pathSvc.toolPath(this.name), + DOTNET_CLI_TELEMETRY_OPTOUT: '1', + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: '1', + }); + } - const nuget = join(this.envSvc.userHome, '.nuget'); - await fs.mkdir(join(nuget, 'NuGet'), { recursive: true }); - // fs isn't recursive, so we use system binaries - await execa('chown', ['-R', this.envSvc.userName, nuget]); - await execa('chmod', ['-R', 'g+w', nuget]); + const nuget = join(this.pathSvc.cachePath, '.nuget'); + if (!(await pathExists(nuget))) { + await this.pathSvc.createDir(join(nuget, 'NuGet')); + } } } diff --git a/src/cli/tools/flutter.ts b/src/cli/tools/flutter.ts index 25fa8142e..3ec2d1f66 100644 --- a/src/cli/tools/flutter.ts +++ b/src/cli/tools/flutter.ts @@ -10,14 +10,21 @@ import { HttpService, PathService, } from '../services'; -import { prepareDartHome, preparePubCache } from './dart/utils'; +import { + initDartHome, + initPubCache, + prepareDartHome, + preparePubCache, +} from './dart/utils'; @injectable() export class FlutterPrepareService extends BasePrepareService { readonly name = 'flutter'; - override async execute(): Promise { + override async prepare(): Promise { + await this.initialize(); await prepareDartHome(this.envSvc, this.pathSvc); + await preparePubCache(this.envSvc, this.pathSvc); // for root await fs.writeFile( @@ -26,24 +33,31 @@ export class FlutterPrepareService extends BasePrepareService { ); // for user - const flutter = join(this.pathSvc.homePath, '.flutter'); + await fs.symlink( + join(this.pathSvc.cachePath, '.flutter'), + join(this.envSvc.userHome, '.flutter'), + ); + + await fs.symlink( + join(this.pathSvc.cachePath, '.flutter_tool_state'), + join(this.envSvc.userHome, '.flutter_tool_state'), + ); + } + + override async initialize(): Promise { + await initDartHome(this.pathSvc); + await initPubCache(this.pathSvc); + + // for user await this.pathSvc.writeFile( - flutter, + join(this.pathSvc.cachePath, '.flutter'), '{ "firstRun": false, "enabled": false }\n', ); - await fs.symlink(flutter, join(this.envSvc.userHome, '.flutter')); - const futterToolState = join(this.pathSvc.homePath, '.flutter_tool_state'); await this.pathSvc.writeFile( - futterToolState, + join(this.pathSvc.cachePath, '.flutter_tool_state'), '{ "is-bot": false, "redisplay-welcome-message": false }\n', ); - await fs.symlink( - futterToolState, - join(this.envSvc.userHome, '.flutter_tool_state'), - ); - - await preparePubCache(this.envSvc, this.pathSvc); } } diff --git a/src/cli/tools/index.ts b/src/cli/tools/index.ts index c7f5770ed..a9d72b333 100644 --- a/src/cli/tools/index.ts +++ b/src/cli/tools/index.ts @@ -34,6 +34,8 @@ export const NoPrepareTools = [ 'yarn-slim', ]; +export const NoInitTools = [...NoPrepareTools]; + /** * Tools in this map are implicit mapped from `install-tool` to `install-`. * So no need for an extra install service. diff --git a/src/cli/tools/java/index.ts b/src/cli/tools/java/index.ts index 22281bcee..32684fcf5 100644 --- a/src/cli/tools/java/index.ts +++ b/src/cli/tools/java/index.ts @@ -32,34 +32,26 @@ export class JavaPrepareService extends BasePrepareService { super(pathSvc, envSvc); } - override async execute(): Promise { - const ssl = this.pathSvc.sslPath; + override async prepare(): Promise { + await this.initialize(); - if (await this.isPrepared()) { + const cacerts = path.join(this.pathSvc.sslPath, 'cacerts'); + + if (await this.pathSvc.fileExists(cacerts)) { // cert store already there return; } - await createMavenSettings(this.pathSvc); - await createGradleSettings(this.pathSvc); - // compatibility with gradle and maven await fs.symlink( - path.join(this.pathSvc.homePath, '.m2'), + path.join(this.pathSvc.cachePath, '.m2'), path.join(this.envSvc.userHome, '.m2'), ); await fs.symlink( - path.join(this.pathSvc.homePath, '.gradle'), + path.join(this.pathSvc.cachePath, '.gradle'), path.join(this.envSvc.userHome, '.gradle'), ); - // fix: Failed to load native library 'libnative-platform.so' for Linux amd64. - await this.pathSvc.exportToolEnv( - 'gradle', - { GRADLE_USER_HOME: path.join(this.pathSvc.homePath, '.gradle') }, - true, - ); - const version = await resolveLatestJavaLtsVersion( this.httpSvc, 'jre', @@ -89,24 +81,33 @@ export class JavaPrepareService extends BasePrepareService { fileName: pkg.name, }); - const tmp = path.join(this.pathSvc.tmpDir, 'java'); + const tmp = path.join(this.envSvc.tmpDir, 'java'); await fs.mkdir(tmp, { recursive: true }); await this.compressionSvc.extract({ file: jre, cwd: tmp, strip: 1 }); - await fs.cp( - path.join(tmp, 'lib/security/cacerts'), - path.join(ssl, 'cacerts'), - ); + const varCerts = path.join(this.pathSvc.varPath, 'cacerts'); + + await fs.cp(path.join(tmp, 'lib/security/cacerts'), varCerts); + + await fs.symlink(varCerts, cacerts); // cleanup will be done by caller } - override async isPrepared(): Promise { - return await this.pathSvc.fileExists( - path.join(this.pathSvc.sslPath, 'cacerts'), - ); + override async initialize(): Promise { + await createMavenSettings(this.pathSvc); + await createGradleSettings(this.pathSvc); + + if (!(await this.pathSvc.toolEnvExists(this.name))) { + // fix: Failed to load native library 'libnative-platform.so' for Linux amd64. + await this.pathSvc.exportToolEnv( + 'gradle', + { GRADLE_USER_HOME: path.join(this.pathSvc.cachePath, '.gradle') }, + true, + ); + } } } diff --git a/src/cli/tools/java/utils.ts b/src/cli/tools/java/utils.ts index f420e90bc..2c9770070 100644 --- a/src/cli/tools/java/utils.ts +++ b/src/cli/tools/java/utils.ts @@ -46,16 +46,16 @@ export async function resolveJavaDownloadUrl( } export async function createMavenSettings(pathSvc: PathService): Promise { - const dir = path.join(pathSvc.homePath, '.m2'); + const dir = path.join(pathSvc.cachePath, '.m2'); + await pathSvc.createDir(dir); + const file = path.join(dir, 'settings.xml'); if (await pathExists(file)) { logger.debug('Maven settings already found'); return; } - logger.debug('Creating Maven settings'); - - await pathSvc.createDir(dir); + logger.debug('Creating Maven settings'); await pathSvc.writeFile( file, codeBlock` @@ -72,16 +72,16 @@ export async function createMavenSettings(pathSvc: PathService): Promise { export async function createGradleSettings( pathSvc: PathService, ): Promise { - const dir = path.join(pathSvc.homePath, '.gradle'); + const dir = path.join(pathSvc.cachePath, '.gradle'); + await pathSvc.createDir(dir); + const file = path.join(dir, 'gradle.properties'); if (await pathExists(file)) { logger.debug('Gradle settings already found'); return; } - logger.debug('Creating Gradle settings'); - - await pathSvc.createDir(dir); + logger.debug('Creating Gradle settings'); await pathSvc.writeFile( file, codeBlock` diff --git a/src/cli/tools/node/index.ts b/src/cli/tools/node/index.ts index 5b33636d5..92c5f2da7 100644 --- a/src/cli/tools/node/index.ts +++ b/src/cli/tools/node/index.ts @@ -22,16 +22,22 @@ import { @injectable() export class NodePrepareService extends BasePrepareService { override name = 'node'; - override async execute(): Promise { + override async prepare(): Promise { + await this.initialize(); + await prepareSymlinks(this.envSvc, this.pathSvc); + } + + override async initialize(): Promise { await prepareNpmCache(this.pathSvc); await prepareNpmrc(this.pathSvc); - await prepareSymlinks(this.envSvc, this.pathSvc); - await this.pathSvc.exportToolEnv(this.name, { - NO_UPDATE_NOTIFIER: '1', - npm_config_update_notifier: 'false', - npm_config_fund: 'false', - }); + if (!(await this.pathSvc.toolEnvExists(this.name))) { + await this.pathSvc.exportToolEnv(this.name, { + NO_UPDATE_NOTIFIER: '1', + npm_config_update_notifier: 'false', + npm_config_fund: 'false', + }); + } } } @@ -121,7 +127,7 @@ export class NodeInstallService extends NodeBaseInstallService { const ver = parse(version); if (ver.major < 15) { const tmp = await fs.mkdtemp( - join(this.pathSvc.tmpDir, 'containerbase-npm-'), + join(this.envSvc.tmpDir, 'containerbase-npm-'), ); const env = this.prepareEnv(version, tmp); env.PATH = `${path}/bin:${penv.PATH}`; diff --git a/src/cli/tools/node/utils.ts b/src/cli/tools/node/utils.ts index af9c10bd7..3ba38b289 100644 --- a/src/cli/tools/node/utils.ts +++ b/src/cli/tools/node/utils.ts @@ -87,7 +87,7 @@ export abstract class NpmBaseInstallService extends NodeBaseInstallService { const nodeVersion = await this.getNodeVersion(); const npm = this.getNodeNpm(nodeVersion); const tmp = await fs.mkdtemp( - join(this.pathSvc.tmpDir, 'containerbase-npm-'), + join(this.envSvc.tmpDir, 'containerbase-npm-'), ); const env = this.prepareEnv(version, tmp); @@ -187,7 +187,12 @@ export abstract class NpmBaseInstallService extends NodeBaseInstallService { } override async test(version: string): Promise { - await execa(this.tool(version), ['--version'], { stdio: 'inherit' }); + let name = this.tool(version); + const idx = name.lastIndexOf('/'); + if (idx > 0) { + name = name.slice(idx + 1); + } + await execa(name, ['--version'], { stdio: 'inherit' }); } override async validate(version: string): Promise { @@ -280,15 +285,13 @@ async function readPackageJson(path: string): Promise { } export async function prepareNpmCache(pathSvc: PathService): Promise { - const path = join(pathSvc.homePath, '.npm'); - if (!(await pathExists(path, true))) { - await pathSvc.createDir(path); - } + const path = join(pathSvc.cachePath, '.npm'); + await pathSvc.createDir(path); } export async function prepareNpmrc(pathSvc: PathService): Promise { - const path = join(pathSvc.homePath, '.npmrc'); - if (!(await pathExists(path, false))) { + const path = join(pathSvc.cachePath, '.npmrc'); + if (!(await pathExists(path))) { await pathSvc.writeFile(path, ''); } } @@ -298,11 +301,11 @@ export async function prepareSymlinks( pathSvc: PathService, ): Promise { await fs.symlink( - join(pathSvc.homePath, '.npm'), + join(pathSvc.cachePath, '.npm'), join(envSvc.userHome, '.npm'), ); await fs.symlink( - join(pathSvc.homePath, '.npmrc'), + join(pathSvc.cachePath, '.npmrc'), join(envSvc.userHome, '.npmrc'), ); } diff --git a/src/cli/tools/php/index.ts b/src/cli/tools/php/index.ts index cd69794e2..3d085f1a7 100644 --- a/src/cli/tools/php/index.ts +++ b/src/cli/tools/php/index.ts @@ -27,7 +27,7 @@ export class PhpPrepareService extends BasePrepareService { super(pathSvc, envSvc); } - override async execute(): Promise { + override async prepare(): Promise { const distro = await getDistro(); switch (distro.versionCode) { diff --git a/src/cli/tools/python/conan.ts b/src/cli/tools/python/conan.ts index 39e436f50..ec3cb4a1f 100644 --- a/src/cli/tools/python/conan.ts +++ b/src/cli/tools/python/conan.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs/promises'; import { join } from 'node:path'; import { codeBlock } from 'common-tags'; import { inject, injectable } from 'inversify'; @@ -19,7 +20,18 @@ export class ConanPrepareService extends BasePrepareService { super(pathSvc, envSvc); } - override async execute(): Promise { + override async prepare(): Promise { + await this.aptSvc.install('cmake', 'gcc', 'g++', 'make', 'perl'); + + await this.initialize(); + + await fs.symlink( + join(this.pathSvc.cachePath, '.conan2'), + join(this.envSvc.userHome, '.conan2'), + ); + } + + override async initialize(): Promise { const distro = await getDistro(); const profile = codeBlock` [settings] @@ -32,10 +44,9 @@ export class ConanPrepareService extends BasePrepareService { os=Linux `; - const profilesPath = join(this.envSvc.userHome, '.conan2', 'profiles'); + const profilesPath = join(this.pathSvc.cachePath, '.conan2', 'profiles'); await this.pathSvc.createDir(profilesPath); await this.pathSvc.writeFile(join(profilesPath, 'default'), profile); - await this.aptSvc.install('cmake', 'gcc', 'g++', 'make', 'perl'); } } diff --git a/src/cli/utils/common.spec.ts b/src/cli/utils/common.spec.ts index 699a99699..37e3d20b3 100644 --- a/src/cli/utils/common.spec.ts +++ b/src/cli/utils/common.spec.ts @@ -7,6 +7,7 @@ import { parseBinaryName, pathExists, reset, + tool2path, validateSystem, } from '.'; import { rootPath } from '~test/path'; @@ -85,11 +86,17 @@ UBUNTU_CODENAME=jammy`); }); test('pathExists', async () => { - fsMocks.stat.mockResolvedValueOnce({ isFile: () => true }); + fsMocks.stat.mockResolvedValueOnce({}); expect(await pathExists('/etc/os-release')).toBe(true); expect(await pathExists('/etc/os-release')).toBe(false); + fsMocks.stat.mockResolvedValueOnce({ isFile: () => true }); + expect(await pathExists('/etc/os-release', 'file')).toBe(true); + expect(await pathExists('/etc/os-release', 'file')).toBe(false); fsMocks.stat.mockResolvedValueOnce({ isDirectory: () => true }); - expect(await pathExists('/etc/os-release', true)).toBe(true); + expect(await pathExists('/etc/os-release', 'dir')).toBe(true); + fsMocks.stat.mockResolvedValueOnce({ isSymbolicLink: () => true }); + expect(await pathExists('/etc/os-release', 'symlink')).toBe(true); + expect(await pathExists('/etc/os-release', 'symlink')).toBe(false); }); test('parseBinaryName', () => { @@ -133,4 +140,8 @@ UBUNTU_CODENAME=jammy`); expect(await isDockerBuild()).toBe(false); }); }); + + test('tool2path', () => { + expect(tool2path('@microsoft/rush//path')).toBe('@microsoft__rush____path'); + }); }); diff --git a/src/cli/utils/common.ts b/src/cli/utils/common.ts index a343ace9d..e80874865 100644 --- a/src/cli/utils/common.ts +++ b/src/cli/utils/common.ts @@ -71,13 +71,23 @@ async function readDistro(): Promise { export const fileRights = fs.constants.S_IRWXU | fs.constants.S_IRWXG | fs.constants.S_IRWXO; +export type PathType = 'file' | 'dir' | 'symlink'; + export async function pathExists( filePath: string, - dir = false, + type?: PathType, ): Promise { try { const fstat = await stat(filePath); - return dir ? fstat.isDirectory() : fstat.isFile(); + switch (type) { + case 'file': + return fstat.isFile(); + case 'dir': + return fstat.isDirectory(); + case 'symlink': + return fstat.isSymbolicLink(); + } + return !!fstat; } catch { return false; } @@ -106,7 +116,7 @@ export async function cleanTmpFiles( tmp: string, dryRun = false, ): Promise { - await deleteAsync([`**`, `!containerbase`], { + await deleteAsync([`**`, `!containerbase/**`], { dot: true, dryRun, force: true, @@ -134,3 +144,7 @@ async function checkDocker(): Promise { export function isDockerBuild(): Promise { return (isDocker ??= checkDocker()); } + +export function tool2path(tool: string): string { + return tool.replace(/\//g, '__'); +} diff --git a/src/usr/local/containerbase/bin/init-tool.sh b/src/usr/local/containerbase/bin/init-tool.sh new file mode 100755 index 000000000..186abe3f7 --- /dev/null +++ b/src/usr/local/containerbase/bin/init-tool.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# shellcheck source=/dev/null +. /usr/local/containerbase/util.sh + + + +V2_TOOL="${CONTAINERBASE_DIR}/tools/v2/${TOOL_NAME}.sh" + +if [[ -f "$V2_TOOL" ]]; then + # init v2 tool + # load overrides needed for v2 tools + # shellcheck source=/dev/null + . "${CONTAINERBASE_DIR}/utils/v2/overrides.sh" + # shellcheck source=/dev/null + . "${V2_TOOL}" + init_v2_tool +else + echo "No tool defined - skipping: ${TOOL_NAME}" >&2 + exit 1; +fi diff --git a/src/usr/local/containerbase/bin/install-tool.sh b/src/usr/local/containerbase/bin/install-tool.sh index 16d103d0e..3753706b9 100755 --- a/src/usr/local/containerbase/bin/install-tool.sh +++ b/src/usr/local/containerbase/bin/install-tool.sh @@ -10,10 +10,6 @@ require_distro require_user require_tool "$@" -if [[ $(ignore_tool) -eq 1 ]]; then - echo "Tool ignored - skipping: ${TOOL_NAME}" - exit 0; -fi TOOL="${CONTAINERBASE_DIR}/tools/${TOOL_NAME}.sh" V2_TOOL="${CONTAINERBASE_DIR}/tools/v2/${TOOL_NAME}.sh" diff --git a/src/usr/local/containerbase/bin/prepare-tool.sh b/src/usr/local/containerbase/bin/prepare-tool.sh index 6155dced7..27c728220 100755 --- a/src/usr/local/containerbase/bin/prepare-tool.sh +++ b/src/usr/local/containerbase/bin/prepare-tool.sh @@ -7,22 +7,4 @@ set -e . /usr/local/containerbase/util.sh require_root - -TOOLS=( "$@" ) -TOOL_PATH="/${CONTAINERBASE_DIR}/tools/v2" - -# special case if only 'all' is given -if [ "$#" -eq 1 ] && [ "${TOOLS[0]}" == "all" ]; then - TOOLS=() - for i in "${TOOL_PATH}"/*.sh; do - TOOLS+=( "$(basename "${i%%.*}")") - done -fi - -for tool in "${TOOLS[@]}" -do - # TODO: find better way to reset env - # shellcheck source=/dev/null - . "/${CONTAINERBASE_DIR}/util.sh" - prepare_tools "${tool}" -done +prepare_tools "${@}" diff --git a/src/usr/local/containerbase/tools/v2/erlang.sh b/src/usr/local/containerbase/tools/v2/erlang.sh index e950ae33b..512dab9da 100644 --- a/src/usr/local/containerbase/tools/v2/erlang.sh +++ b/src/usr/local/containerbase/tools/v2/erlang.sh @@ -2,6 +2,8 @@ SEMVER_REGEX_ERLANG="^(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))?(\.(0|[1-9][0-9]*))?(\.(0|[1-9][0-9]*))?(\+[0-9]+)?([a-z-].*)?$" +export NEEDS_PREPARE=1 + function check_semver_erlang () { if [[ ! "${1}" =~ ${SEMVER_REGEX_ERLANG} ]]; then echo Not a semver like version - aborting: "${1}" @@ -35,7 +37,7 @@ function check_tool_requirements () { function prepare_tool() { local tool_path - tool_path=$(create_tool_path) + tool_path=$(find_tool_path) # Workaround for compatibillity for Erlang hardcoded paths, works for v22+ if [ "${tool_path}" != "${ROOT_DIR_LEGACY}/erlang" ]; then ln -sf "${tool_path}" /usr/local/erlang @@ -56,15 +58,6 @@ function install_tool () { tool_path=$(find_tool_path) - if [[ ! -d "${tool_path}" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${name} not prepared" >&2 - exit 1 - fi - prepare_tool - tool_path=$(find_tool_path) - fi - base_url="https://github.com/containerbase/${name}-prebuild/releases/download" version_codename=$(get_distro) diff --git a/src/usr/local/containerbase/tools/v2/golang.sh b/src/usr/local/containerbase/tools/v2/golang.sh index bc340f0ba..bb3efbdac 100644 --- a/src/usr/local/containerbase/tools/v2/golang.sh +++ b/src/usr/local/containerbase/tools/v2/golang.sh @@ -1,19 +1,32 @@ #!/bin/bash +export NEEDS_PREPARE=1 function prepare_tool() { + local go_path # go suggests: git svn bzr mercurial apt_install bzr mercurial - go_path=$(get_home_path)/go + init_tool + + go_path=$(get_cache_path)/go ln -sf "${go_path}" "${USER_HOME}/go" +} + +function init_tool() { + local go_path + + go_path=$(get_cache_path)/go - mkdir -p "${go_path}/src" "${go_path}/bin" "${go_path}/pkg" + if [ -d "${go_path}" ]; then + return + fi + create_folder "${go_path}/src" 775 + create_folder "${go_path}/bin" 775 + create_folder "${go_path}/pkg" 775 chown -R "${USER_ID}" "${go_path}" - chmod -R g+w "${go_path}" - create_tool_path > /dev/null } function install_tool () { @@ -27,14 +40,6 @@ function install_tool () { local version=${TOOL_VERSION} local versioned_tool_path - if [[ ! -d "$(find_tool_path)" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${TOOL_NAME} not prepared" - exit 1 - fi - prepare_tool - fi - base_url="https://github.com/containerbase/${name}-prebuild/releases/download" # not all releases are copied to github diff --git a/src/usr/local/containerbase/tools/v2/nix.sh b/src/usr/local/containerbase/tools/v2/nix.sh index 114b3abb5..d84b09638 100644 --- a/src/usr/local/containerbase/tools/v2/nix.sh +++ b/src/usr/local/containerbase/tools/v2/nix.sh @@ -35,6 +35,6 @@ function link_tool() { local versioned_tool_path versioned_tool_path=$(find_versioned_tool_path) - shell_wrapper "${TOOL_NAME}" "${versioned_tool_path}/bin" "NIX_STORE_DIR=$(get_home_path)/nix/store NIX_DATA_DIR=$(get_home_path)/nix/data NIX_LOG_DIR=$(get_cache_path)/nix/log NIX_STATE_DIR=$(get_home_path)/nix/state NIX_CONF_DIR=$(get_home_path)/nix/conf" + shell_wrapper "${TOOL_NAME}" "${versioned_tool_path}/bin" "NIX_STORE_DIR=$(get_cache_path)/nix/store NIX_DATA_DIR=$(get_cache_path)/nix/data NIX_LOG_DIR=$(get_cache_path)/nix/log NIX_STATE_DIR=$(get_cache_path)/nix/state NIX_CONF_DIR=$(get_cache_path)/nix/conf" [[ -n $SKIP_VERSION ]] || nix --version } diff --git a/src/usr/local/containerbase/tools/v2/powershell.sh b/src/usr/local/containerbase/tools/v2/powershell.sh index 50f85677d..c20c6c4da 100644 --- a/src/usr/local/containerbase/tools/v2/powershell.sh +++ b/src/usr/local/containerbase/tools/v2/powershell.sh @@ -1,5 +1,7 @@ #!/bin/bash +export NEEDS_PREPARE=1 + function prepare_tool() { local version_codename @@ -13,8 +15,6 @@ function prepare_tool() { exit 1 ;; esac - - create_tool_path > /dev/null } function install_tool () { @@ -22,14 +22,6 @@ function install_tool () { local versioned_tool_path local arch=linux-x64 - if [[ ! -d "$(find_tool_path)" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${TOOL_NAME} not prepared" - exit 1 - fi - prepare_tool - fi - if [[ "$(uname -p)" = "aarch64" ]]; then arch=linux-arm64 fi diff --git a/src/usr/local/containerbase/tools/v2/python.sh b/src/usr/local/containerbase/tools/v2/python.sh index eed040bc1..da6281d95 100644 --- a/src/usr/local/containerbase/tools/v2/python.sh +++ b/src/usr/local/containerbase/tools/v2/python.sh @@ -1,5 +1,7 @@ #!/bin/bash +export NEEDS_PREPARE=1 + # sets the correct shebang for python fix_python_shebangs() { # https://github.com/koalaman/shellcheck/wiki/SC2044 @@ -40,7 +42,7 @@ function prepare_tool() { libpq-dev \ ; - tool_path=$(create_tool_path) + tool_path=$(find_tool_path) # Workaround for compatibillity for Python hardcoded paths if [ "${tool_path}" != "${ROOT_DIR_LEGACY}/python" ]; then @@ -65,15 +67,6 @@ function install_tool () { tool_path=$(find_tool_path) - if [[ ! -d "${tool_path}" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${name} not prepared" - exit 1 - fi - prepare_tool - tool_path=$(find_tool_path) - fi - base_url="https://github.com/containerbase/${name}-prebuild/releases/download" version_codename=$(get_distro) diff --git a/src/usr/local/containerbase/tools/v2/ruby.sh b/src/usr/local/containerbase/tools/v2/ruby.sh index 12638a234..30d573065 100644 --- a/src/usr/local/containerbase/tools/v2/ruby.sh +++ b/src/usr/local/containerbase/tools/v2/ruby.sh @@ -1,8 +1,9 @@ #!/bin/bash +export NEEDS_PREPARE=1 + function prepare_tool() { local version_codename - local tool_path local path version_codename="$(get_distro)" @@ -21,42 +22,61 @@ function prepare_tool() { libyaml-0-2 \ make \ ; - tool_path=$(create_tool_path) + + init_tool # Redirect gemrc - path="$(get_home_path)/.gemrc" + path="$(get_cache_path)/.gemrc" + ln -sf "${path}" "${USER_HOME}/.gemrc" + + # Redirect gem home + path="$(get_cache_path)/.gem" + ln -sf "${path}" "${USER_HOME}/.gem" + + # Redirect cocoapods home + path="$(get_cache_path)/.cocoapods" + ln -sf "${path}" "${USER_HOME}/.cocoapods" + + # Redirect Library home + path="$(get_cache_path)/Library" + ln -sf "${path}" "${USER_HOME}/Library" + + # Workaround for compatibillity for Ruby hardcoded paths + path=$(find_tool_path) + if [ "${path}" != "${ROOT_DIR_LEGACY}/ruby" ]; then + ln -sf "${path}" /usr/local/ruby + fi +} + +function init_tool () { + local path + path="$(get_cache_path)/.gemrc" + + if [ -f "${path}" ]; then + return + fi + + # Init gemrc { printf -- "gem: --no-document\n" } > "${path}" chown "${USER_ID}" "${path}" chmod g+w "${path}" - ln -sf "${path}" "${USER_HOME}/.gemrc" - # Redirect gem home - path="$(get_home_path)/.gem" + # Init gem home + path="$(get_cache_path)/.gem" create_folder "${path}" 775 chown "${USER_ID}" "${path}" - chmod g+w "${path}" - ln -sf "${path}" "${USER_HOME}/.gem" - # Redirect cocoapods home - path="$(get_home_path)/.cocoapods" + # Init cocoapods home + path="$(get_cache_path)/.cocoapods" create_folder "${path}" 775 chown "${USER_ID}" "${path}" - chmod g+w "${path}" - ln -sf "${path}" "${USER_HOME}/.cocoapods" - # Redirect Library home - path="$(get_home_path)/Library" + # Init Library home + path="$(get_cache_path)/Library" create_folder "${path}" 775 chown "${USER_ID}" "${path}" - chmod g+w "${path}" - ln -sf "${path}" "${USER_HOME}/Library" - - # Workaround for compatibillity for Ruby hardcoded paths - if [ "${tool_path}" != "${ROOT_DIR_LEGACY}/ruby" ]; then - ln -sf "${tool_path}" /usr/local/ruby - fi } function install_tool () { @@ -73,15 +93,6 @@ function install_tool () { tool_path=$(find_tool_path) - if [[ ! -d "${tool_path}" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${TOOL_NAME} not prepared" - exit 1 - fi - prepare_tool - tool_path=$(find_tool_path) - fi - arch=$(uname -p) base_url="https://github.com/containerbase/${name}-prebuild/releases/download" version_codename=$(get_distro) diff --git a/src/usr/local/containerbase/tools/v2/rust.sh b/src/usr/local/containerbase/tools/v2/rust.sh index 02336099e..75e3cd7c7 100644 --- a/src/usr/local/containerbase/tools/v2/rust.sh +++ b/src/usr/local/containerbase/tools/v2/rust.sh @@ -1,15 +1,28 @@ #!/bin/bash +export NEEDS_PREPARE=1 + function prepare_tool() { local cargo_home - cargo_home=$(get_home_path)/.cargo + init_tool + cargo_home=$(get_cache_path)/.cargo ln -sf "${cargo_home}" "${USER_HOME}/.cargo" - mkdir "${cargo_home}" +} + +function init_tool () { + local cargo_home + + cargo_home=$(get_cache_path)/.cargo + + if [ -d "${cargo_home}" ]; then + # already initialized + return + fi + + create_folder "${cargo_home}" 775 chown -R "${USER_ID}" "${cargo_home}" - chmod -R g+w "${cargo_home}" - create_tool_path > /dev/null } function install_tool () { @@ -21,18 +34,6 @@ function install_tool () { local expected_checksum local ext=gz local file_name - local tool_path - - tool_path=$(find_tool_path) - - if [[ ! -d "${tool_path}" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${TOOL_NAME} not prepared" - exit 1 - fi - prepare_tool - tool_path=$(find_tool_path) - fi arch=$(uname -p) file_name="rust-${TOOL_VERSION}-${arch}-unknown-linux-gnu.tar" diff --git a/src/usr/local/containerbase/tools/v2/swift.sh b/src/usr/local/containerbase/tools/v2/swift.sh index ed1ec7946..1f216afbd 100644 --- a/src/usr/local/containerbase/tools/v2/swift.sh +++ b/src/usr/local/containerbase/tools/v2/swift.sh @@ -1,5 +1,7 @@ #!/bin/bash +export NEEDS_PREPARE=1 + function prepare_tool() { local version_codename local tool_path @@ -66,8 +68,6 @@ function prepare_tool() { exit 1 ;; esac - - create_tool_path > /dev/null } function install_tool () { @@ -85,16 +85,6 @@ function install_tool () { tool_path=$(find_tool_path) - if [[ ! -d "${tool_path}" ]]; then - if [[ $(is_root) -ne 0 ]]; then - echo "${TOOL_NAME} not prepared" - exit 1 - fi - prepare_tool - tool_path=$(find_tool_path) - fi - - # shellcheck source=/dev/null VERSION_ID=$(. /etc/os-release && echo "${VERSION_ID}") diff --git a/src/usr/local/containerbase/util.sh b/src/usr/local/containerbase/util.sh index 48f56c433..8b62c5af9 100644 --- a/src/usr/local/containerbase/util.sh +++ b/src/usr/local/containerbase/util.sh @@ -20,6 +20,8 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi # shellcheck source=/dev/null . "${DIR}/utils/install.sh" # shellcheck source=/dev/null +. "${DIR}/utils/init.sh" +# shellcheck source=/dev/null . "${DIR}/utils/prepare.sh" # shellcheck source=/dev/null . "${DIR}/utils/user.sh" diff --git a/src/usr/local/containerbase/utils/constants.sh b/src/usr/local/containerbase/utils/constants.sh index 26c032b94..f5ae77c30 100644 --- a/src/usr/local/containerbase/utils/constants.sh +++ b/src/usr/local/containerbase/utils/constants.sh @@ -29,6 +29,9 @@ export TEMP_DIR=/tmp # used to source helper from tools export CONTAINERBASE_DIR=/usr/local/containerbase +export CONTAINERBASE_VAR_DIR=/var/lib/containerbase +export CONTAINERBASE_TMP_DIR=/tmp/containerbase + # Used to find matching tool downloads ARCHITECTURE=$(uname -p) export ARCHITECTURE diff --git a/src/usr/local/containerbase/utils/filesystem.sh b/src/usr/local/containerbase/utils/filesystem.sh index 8c10129bf..9cc71a866 100644 --- a/src/usr/local/containerbase/utils/filesystem.sh +++ b/src/usr/local/containerbase/utils/filesystem.sh @@ -11,6 +11,7 @@ function get_install_dir () { function find_tool_path () { local tool=${1:-$TOOL_NAME} + tool=${tool//\//__} install_dir=$(get_install_dir) if [[ -d "${install_dir}/${tool}" ]]; then echo "${install_dir}/${tool}" @@ -20,6 +21,7 @@ function find_tool_path () { function find_versioned_tool_path () { local tool=${1:-$TOOL_NAME} local version=${2:-$TOOL_VERSION} + tool=${tool//\//__} tool_dir=$(find_tool_path "$tool") if [[ -d "${tool_dir}/${version}" ]]; then echo "${tool_dir}/${version}" @@ -28,6 +30,7 @@ function find_versioned_tool_path () { function create_versioned_tool_path () { local tool=${1:-$TOOL_NAME} + tool=${tool//\//__} install_dir=$(get_install_dir) local umask=775 @@ -44,11 +47,12 @@ function create_versioned_tool_path () { # Will set up the general folder structure for the whole containerbase installation function setup_directories () { local install_dir - local home_path + local cache_path install_dir=$(get_install_dir) - home_path=$(get_home_path) + cache_path=$(get_cache_path) - mkdir -p "${install_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 775 "${install_dir}" # contains the installed tools # shellcheck disable=SC2174 mkdir -p -m 775 "$(get_tools_path)" @@ -67,14 +71,12 @@ function setup_directories () { # contains the certificates for the tools # shellcheck disable=SC2174 mkdir -p -m 775 "$(get_ssl_path)" - # contains the caches for the tools - # shellcheck disable=SC2174 - mkdir -p -m 775 "$(get_cache_path)" - # contains the home for the tools - # shellcheck disable=SC2174 - mkdir -p -m 775 "${home_path}" - # shellcheck disable=SC2174 - mkdir -p -m 775 "${home_path}"/{.cache,.config,.local} + + create_folder "$(get_tool_init_path)" 775 + + create_folder "$(get_tool_prep_path)" 755 + + create_folder "$(get_containerbase_tmp_path)" 775 # symlink v2 tools bin and lib rm -rf "${BIN_DIR}" "${LIB_DIR}" @@ -82,9 +84,9 @@ function setup_directories () { ln -sf "${ROOT_DIR}/lib" "${LIB_DIR}" # symlink known user folders - ln -sf "${home_path}/.config" "${USER_HOME}/.config" - ln -sf "${home_path}/.local" "${USER_HOME}/.local" - ln -sf "$(get_cache_path)" "${USER_HOME}/.cache" + ln -sf "${cache_path}/.config" "${USER_HOME}/.config" + ln -sf "${cache_path}/.local" "${USER_HOME}/.local" + ln -sf "${cache_path}/.cache" "${USER_HOME}/.cache" } # Creates the given folder path with root and user umask depending on the caller @@ -92,17 +94,17 @@ function setup_directories () { # The umask can be provided with the second argument function create_folder () { local folder=${1} + local parent + check folder local umask=${2:-"$(get_umask)"} - local parent - parent=$(dirname "${folder}") - if [ -d "${folder}" ]; then return fi + parent=$(dirname "${folder}") if [ ! -d "${parent}" ]; then create_folder "$parent" fi @@ -145,16 +147,15 @@ function get_ssl_path () { # Gets the path to the cache folder function get_cache_path () { - local install_dir - install_dir=$(get_install_dir) - echo "${install_dir}/cache" -} - -# Gets the path to the home folder -function get_home_path () { - local install_dir - install_dir=$(get_install_dir) - echo "${install_dir}/home" + local cache_path + cache_path="$(get_containerbase_tmp_path)/cache" + if [[ ! -d "${cache_path}" ]]; then + create_folder "${cache_path}" 775 + create_folder "${cache_path}/.cache" 775 + create_folder "${cache_path}/.config" 775 + create_folder "${cache_path}/.local" 775 + fi + echo "${cache_path}" } # will get the correct umask based on the caller id @@ -172,6 +173,63 @@ function get_containerbase_path () { echo "${CONTAINERBASE_DIR}" } +# Gets the path to the var folder to persist prepared tools state +function get_containerbase_var_path () { + create_folder "${CONTAINERBASE_VAR_DIR}" 755 + echo "${CONTAINERBASE_VAR_DIR}" +} + +# Gets the path to the tmp folder to persist tool files +function get_containerbase_tmp_path () { + create_folder "${CONTAINERBASE_TMP_DIR}" 775 + echo "${CONTAINERBASE_TMP_DIR}" +} + +# Gets the path to the tool prep state folder +function get_tool_prep_path () { + echo "$(get_containerbase_var_path)/tool.prep.d" +} + +function set_tool_prep () { + local tool=${1:-${TOOL_NAME}} + tool=${tool//\//__} + if [[ ! -f "$(get_tool_prep_path)/${tool}" ]]; then + touch "$(get_tool_prep_path)/${tool}" + fi +} + +function get_tool_prep () { + local tool=${1:-${TOOL_NAME}} + tool=${tool//\//__} + if [[ -f "$(get_tool_prep_path)/${tool}" ]]; then + echo "$(get_tool_prep_path)/${tool}" + fi +} + +# Gets the path to the tool init state folder +function get_tool_init_path () { + if [[ ! -d "$(get_containerbase_tmp_path)/tool.init.d" ]]; then + create_folder "$(get_containerbase_tmp_path)/tool.init.d" 775 + fi + echo "$(get_containerbase_tmp_path)/tool.init.d" +} + +function set_tool_init () { + local tool=${1:-${TOOL_NAME}} + tool=${tool//\//__} + if [[ ! -f "$(get_tool_init_path)/${tool}" ]]; then + touch "$(get_tool_init_path)/${tool}" + fi +} + +function get_tool_init () { + local tool=${1:-${TOOL_NAME}} + tool=${tool//\//__} + if [[ -f "$(get_tool_init_path)/${tool}" ]]; then + echo "$(get_tool_init_path)/${tool}" + fi +} + # Own the file by default user and make it writable for root group function set_file_owner() { local target=${1} diff --git a/src/usr/local/containerbase/utils/init.sh b/src/usr/local/containerbase/utils/init.sh new file mode 100644 index 000000000..30b49f1b9 --- /dev/null +++ b/src/usr/local/containerbase/utils/init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function init_v2_tool () { + + if [[ -f "$(get_tool_init)" ]]; then + # tool already initialized + return + fi + + # ensure tool path exists + create_tool_path > /dev/null + + # init tool + init_tool + + set_tool_init +} diff --git a/src/usr/local/containerbase/utils/install.sh b/src/usr/local/containerbase/utils/install.sh index 72b3bf5b1..74b18dbd4 100644 --- a/src/usr/local/containerbase/utils/install.sh +++ b/src/usr/local/containerbase/utils/install.sh @@ -7,11 +7,22 @@ function install_v2_tool () { # load overrides needed for v2 tools # shellcheck source=/dev/null - . "/${CONTAINERBASE_DIR}/utils/v2/overrides.sh" + . "${CONTAINERBASE_DIR}/utils/v2/overrides.sh" # shellcheck source=/dev/null . "${path}" + if [[ ! -f "$(get_tool_prep)" && "${NEEDS_PREPARE}" -eq 1 ]]; then + if [[ $(is_root) -ne 0 ]]; then + echo "${TOOL_NAME} not prepared" + exit 1 + fi + prepare_tool_wrapper + fi + + # init tool if required + init_v2_tool + check_tool_requirements if ! check_tool_installed; then diff --git a/src/usr/local/containerbase/utils/linking.sh b/src/usr/local/containerbase/utils/linking.sh index b480b0e1e..930e07e7a 100644 --- a/src/usr/local/containerbase/utils/linking.sh +++ b/src/usr/local/containerbase/utils/linking.sh @@ -7,6 +7,7 @@ function shell_wrapper () { local EXPORTS=$3 local args=$4 local content=$5 + local tool_init TARGET="$(get_bin_path)/${1}" if [[ -z "$SOURCE" ]]; then SOURCE=$(command -v "${1}") @@ -17,12 +18,18 @@ function shell_wrapper () { check SOURCE true check_command "$SOURCE" + tool_init=$(get_tool_init "${TOOL_NAME//\//__}") + cat > "$TARGET" <<- EOM #!/bin/bash if [[ -z "\${CONTAINERBASE_ENV+x}" ]]; then . $ENV_FILE fi +if [[ ! -f "${tool_init}" ]]; then + # set logging to only warn and above to not interfere with tool output + CONTAINERBASE_LOG_LEVEL=warn containerbase-cli init tool "${TOOL_NAME}" +fi EOM if [ -n "$EXPORTS" ]; then diff --git a/src/usr/local/containerbase/utils/prepare.sh b/src/usr/local/containerbase/utils/prepare.sh index af5bb9d5b..3f437ae58 100644 --- a/src/usr/local/containerbase/utils/prepare.sh +++ b/src/usr/local/containerbase/utils/prepare.sh @@ -27,6 +27,27 @@ function prepare_tools () { exit 1 fi + prepare_tool_wrapper +} + +function prepare_tool_wrapper () { + # force root check + require_root + + if [[ -f "$(get_tool_prep)" ]]; then + # tool already prepared + return + fi + + # ensure tool path exists + create_tool_path > /dev/null + # prepare tool prepare_tool + + # set tool preped + set_tool_prep + + # init tool + init_v2_tool } diff --git a/src/usr/local/containerbase/utils/v2/defaults.sh b/src/usr/local/containerbase/utils/v2/defaults.sh index 055a20811..78b615080 100644 --- a/src/usr/local/containerbase/utils/v2/defaults.sh +++ b/src/usr/local/containerbase/utils/v2/defaults.sh @@ -35,7 +35,12 @@ function link_tool () { # Installs needed packages to make the tool runtime installable function prepare_tool() { - echo "'prepare_tool' not defined for tool ${TOOL_NAME}" + true +} + +# creates required files and folders for the tool +function init_tool() { + true } # Called after install_tool and link_tool. It's always called. diff --git a/test/bash/filesystem.bats b/test/bash/filesystem.bats index 5d137ad71..c7eff7d13 100644 --- a/test/bash/filesystem.bats +++ b/test/bash/filesystem.bats @@ -115,6 +115,8 @@ teardown() { @test "setup directories with correct permissions" { local TEST_ROOT_USER=0 # root local install_dir=$(get_install_dir) + local tmp_dir=$(get_containerbase_tmp_path) + local var_dir=$(get_containerbase_var_path) run setup_directories assert_success @@ -130,10 +132,10 @@ teardown() { assert [ "$(stat --format '%a' "${BIN_DIR}")" -eq 777 ] assert [ -d "${install_dir}/env.d" ] assert [ "$(stat --format '%a' "${install_dir}/env.d")" -eq 775 ] - assert [ -d "${install_dir}/cache" ] - assert [ "$(stat --format '%a' "${install_dir}/cache")" -eq 775 ] - assert [ -d "${install_dir}/home" ] - assert [ "$(stat --format '%a' "${install_dir}/home")" -eq 775 ] + assert [ -d "${tmp_dir}/cache" ] + assert [ "$(stat --format '%a' "${tmp_dir}/cache")" -eq 775 ] + assert [ -d "${var_dir}/tool.prep.d" ] + assert [ "$(stat --format '%a' "${var_dir}/tool.prep.d")" -eq 755 ] } @test "creates a folder with correct permissions" { diff --git a/test/bash/util.sh b/test/bash/util.sh index a0f3ab3e8..287b25c12 100644 --- a/test/bash/util.sh +++ b/test/bash/util.sh @@ -10,6 +10,8 @@ export BIN_DIR="${TEST_ROOT_DIR}/bin" export LIB_DIR="${TEST_ROOT_DIR}/lib" export USER_HOME="${TEST_ROOT_DIR}/user" export ENV_FILE="${TEST_ROOT_DIR}/env" +export CONTAINERBASE_VAR_DIR="${TEST_ROOT_DIR}/var" +export CONTAINERBASE_TMP_DIR="${TEST_ROOT_DIR}/tmp" # set default test user export TEST_ROOT_USER=12021 diff --git a/test/bash/v2/defaults.bats b/test/bash/v2/defaults.bats index 14c2d1352..daea2f1d2 100644 --- a/test/bash/v2/defaults.bats +++ b/test/bash/v2/defaults.bats @@ -58,5 +58,4 @@ teardown() { run prepare_tool assert_success - assert_output --partial "not defined" } diff --git a/test/java/Dockerfile b/test/java/Dockerfile index 325798c84..dcca46c87 100644 --- a/test/java/Dockerfile +++ b/test/java/Dockerfile @@ -51,23 +51,22 @@ RUN set -ex; \ [ $(stat --format '%a' "${USER_HOME}/.m2") -eq 777 ]; \ true -RUN [ "$(find /tmp -type f | wc -l)" -eq 0 ] - RUN set -ex; \ ls -la /tmp; \ - find /tmp -type f; \ - exit 0 + find /tmp -mindepth 1 -maxdepth 1; \ + true + +RUN [ "$(find /tmp -mindepth 1 -maxdepth 1 | wc -l)" -eq 1 ] # renovate: datasource=java-version packageName=java-jre RUN install-tool java 21.0.4+7.0.LTS RUN set -ex; \ ls -la /tmp; \ - find /tmp -type f; \ - exit 0 + true -RUN [ "$(find /tmp -type f | wc -l)" -eq 0 ] +RUN [ "$(find /tmp -mindepth 1 -maxdepth 1 | wc -l)" -eq 1 ] #-------------------------------------- # test: Java 11 LTS + Gradle 6 @@ -138,8 +137,8 @@ RUN install-tool gradle 7.6.1 RUN set -ex; \ ls -la /root/; \ ls -la /opt/containerbase/; \ - ls -la /opt/containerbase/home/; \ - ls -la /opt/containerbase/home/.gradle; \ + ls -la /tmp/containerbase/cache/; \ + ls -la /tmp/containerbase/cache/.gradle; \ true # test openshift userid missmatch diff --git a/test/node/Dockerfile b/test/node/Dockerfile index 277ccf288..37525252b 100644 --- a/test/node/Dockerfile +++ b/test/node/Dockerfile @@ -44,7 +44,7 @@ FROM base AS build RUN install-tool node 20.17.0 # ensure npmrc is writable by user -RUN set -ex; [ $(stat --format '%u' "/opt/containerbase/home/.npmrc") -eq ${USER_ID} ] +RUN set -ex; [ $(stat --format '%u' "/tmp/containerbase/cache/.npmrc") -eq ${USER_ID} ] USER 12021 @@ -466,12 +466,29 @@ RUN set -ex; \ yarn -v | grep ${YARN_SLIM_VERSION}; #-------------------------------------- -# test: renovate +# test: renovate (rootless,readonly) #-------------------------------------- FROM base AS testq +RUN set -ex; \ + ls -la /opt/containerbase; \ + ls -la /tmp/containerbase; \ + ls -la /tmp/containerbase/cache; \ + true;true + RUN prepare-tool node +# emulate emtpy containerbase folders +RUN set -ex; \ + ls -la /tmp/containerbase/cache; \ + rm -rf /opt/containerbase/*; \ + ls -la /opt/containerbase/; \ + rm -rf /tmp/*; \ + ls -la /tmp/; \ + true + +USER 1000 + # install latest version RUN install-tool node RUN install-tool yarn @@ -500,20 +517,20 @@ RUN set -ex; \ #-------------------------------------- FROM base -COPY --from=testa /.dummy /.dummy -COPY --from=testb /.dummy /.dummy -COPY --from=testc /.dummy /.dummy -COPY --from=testd /.dummy /.dummy -COPY --from=teste /.dummy /.dummy -COPY --from=testf /.dummy /.dummy -COPY --from=testg /.dummy /.dummy -COPY --from=testh /.dummy /.dummy -COPY --from=testi /.dummy /.dummy -COPY --from=testj /.dummy /.dummy -COPY --from=testk /.dummy /.dummy -COPY --from=testl /.dummy /.dummy -COPY --from=testm /.dummy /.dummy -COPY --from=testn /.dummy /.dummy -COPY --from=testo /.dummy /.dummy -COPY --from=testp /.dummy /.dummy +# COPY --from=testa /.dummy /.dummy +# COPY --from=testb /.dummy /.dummy +# COPY --from=testc /.dummy /.dummy +# COPY --from=testd /.dummy /.dummy +# COPY --from=teste /.dummy /.dummy +# COPY --from=testf /.dummy /.dummy +# COPY --from=testg /.dummy /.dummy +# COPY --from=testh /.dummy /.dummy +# COPY --from=testi /.dummy /.dummy +# COPY --from=testj /.dummy /.dummy +# COPY --from=testk /.dummy /.dummy +# COPY --from=testl /.dummy /.dummy +# COPY --from=testm /.dummy /.dummy +# COPY --from=testn /.dummy /.dummy +# COPY --from=testo /.dummy /.dummy +# COPY --from=testp /.dummy /.dummy COPY --from=testq /.dummy /.dummy diff --git a/test/rust/Dockerfile b/test/rust/Dockerfile index 52e5067e4..927ce68df 100644 --- a/test/rust/Dockerfile +++ b/test/rust/Dockerfile @@ -85,6 +85,11 @@ SHELL [ "/bin/sh", "-c" ] RUN rustc --version RUN cargo --version +RUN set -ex; \ + ls -la /tmp/containerbase; \ + ls -la /tmp/containerbase/cache; \ + true + #-------------------------------------- # final #--------------------------------------