diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4fde1620..4099bfb3d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,6 +86,9 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} + - name: build + run: yarn build + - name: bats run: yarn test:bats diff --git a/README.md b/README.md index b4727d1d9..11a9469ac 100644 --- a/README.md +++ b/README.md @@ -98,10 +98,9 @@ You can replace the default urls used to download the tools. The number of `URL_REPLACE_*_FROM` and `URL_REPLACE_*_TO` environment variables must match. Theses variables are case sensitive. The numbers will be processed in numerical order and can have gaps. -This is currently only supported by these tool installers: - -- `docker` -- `dart` +Almost all tools are now supported. +Only tools which are installed by `gem`, `npm` or `pip` are not supported. +They can be configured via thier own environment variables. Checkout [#1067](https://github.com/containerbase/base/issues/1067) for additional support. diff --git a/src/cli/command/download-file.ts b/src/cli/command/download-file.ts new file mode 100644 index 000000000..9d9034572 --- /dev/null +++ b/src/cli/command/download-file.ts @@ -0,0 +1,54 @@ +import { createWriteStream } from 'node:fs'; +import { mkdir } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import { Command, Option } from 'clipanion'; +import { got } from 'got'; +import prettyMilliseconds from 'pretty-ms'; +import { EnvService, rootContainer } from '../services'; +import { logger } from '../utils'; + +export class DownloadFileCommand extends Command { + static override paths = [['download', 'file'], ['df']]; + + static override usage = Command.Usage({ + description: 'Downloads a file and optionally validates the checksum.', + }); + + url = Option.String({ required: true }); + output = Option.String({ required: true }); + + // checksum = Option.String('-c,--checksum'); + // algo = Option.String('-a,--algo'); + + // dryRun = Option.Boolean('-d,--dry-run', false); + + async execute(): Promise { + const start = Date.now(); + let error = false; + logger.info({ url: this.url, output: this.output }, `Downloading file ...`); + try { + const container = rootContainer.createChild(); + + const env = container.get(EnvService); + const path = dirname(this.output); + + await mkdir(path, { recursive: true }); + + const nUrl = env.replaceUrl(this.url); + await pipeline(got.stream(nUrl), createWriteStream(this.output)); + + return 0; + } catch (err) { + logger.fatal(err); + error = true; + return 1; + } finally { + logger.info( + `Download completed ${ + error ? 'with errors ' : '' + } in ${prettyMilliseconds(Date.now() - start)}.` + ); + } + } +} diff --git a/src/cli/command/index.ts b/src/cli/command/index.ts index 10c2ab117..cec7525e9 100644 --- a/src/cli/command/index.ts +++ b/src/cli/command/index.ts @@ -2,6 +2,7 @@ import { argv0 } from 'node:process'; import type { Cli } from 'clipanion'; import type { CliMode } from '../utils'; import { logger } from '../utils/logger'; +import { DownloadFileCommand } from './download-file'; import { InstallToolCommand, InstallToolShortCommand } from './install-tool'; import { PrepareToolCommand, PrepareToolShortCommand } from './prepare-tool'; import { prepareToolVersion } from './utils'; @@ -28,6 +29,7 @@ export function prepareCommands( cli.register(InstallToolCommand); cli.register(PrepareToolCommand); + cli.register(DownloadFileCommand); return prepareToolVersion(mode, args); } diff --git a/src/usr/local/containerbase/util.sh b/src/usr/local/containerbase/util.sh index 39527e59d..50fb45eb6 100644 --- a/src/usr/local/containerbase/util.sh +++ b/src/usr/local/containerbase/util.sh @@ -1,5 +1,8 @@ #!/bin/bash +# https://github.com/vercel/pkg/issues/1861 +unset PKG_EXECPATH + # get path location DIR="${BASH_SOURCE%/*}" if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi diff --git a/src/usr/local/containerbase/utils/cache.sh b/src/usr/local/containerbase/utils/cache.sh index 1aa21de3c..ade673705 100644 --- a/src/usr/local/containerbase/utils/cache.sh +++ b/src/usr/local/containerbase/utils/cache.sh @@ -90,7 +90,7 @@ function download_file () { local temp_folder=${CONTAINERBASE_CACHE_DIR:-${TEMP_DIR}} while [ "${retry}" -gt 0 ]; do retry=$((retry-1)) - if ! curl --retry 3 --create-dirs -sSfLo "${temp_folder}/${name}" "${url}" ; then + if ! containerbase-cli df "${url}" "${temp_folder}/${name}" >&2; then echo "Download failed: ${url}" >&2 exit 1 fi; @@ -119,7 +119,7 @@ function download_file () { # First argument is the path to the file # Second argument is the checksum # Third argument is the type, currently supported: -# * sha1 +# * sha1sum # * sha224sum # * sha256sum # * sha384sum diff --git a/test/bash/cache.bats b/test/bash/cache.bats index 61a0abfe3..9c98aaafe 100644 --- a/test/bash/cache.bats +++ b/test/bash/cache.bats @@ -156,11 +156,11 @@ teardown() { run download_file "${file}" assert_success - assert_output "${CONTAINERBASE_CACHE_DIR}/containerbase.tar.xz" + assert_line "${CONTAINERBASE_CACHE_DIR}/containerbase.tar.xz" run download_file "${file}" "foobar" assert_success - assert_output "${CONTAINERBASE_CACHE_DIR}/foobar" + assert_line "${CONTAINERBASE_CACHE_DIR}/foobar" CONTAINERBASE_CACHE_DIR= \ tmp_file=$(download_file "${file}") @@ -178,11 +178,11 @@ teardown() { run get_from_url "${file}" assert_success - assert_output --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz" + assert_line --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz" run get_from_url "${file}" test assert_success - assert_output --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test" + assert_line --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test" # overwrite donwload function to fail function download_file () { @@ -209,13 +209,13 @@ teardown() { run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha512sum" assert_success - assert_output --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz" + assert_line --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz" rm -rf "${CONTAINERBASE_CACHE_DIR}" run get_from_url "${file}" test "${checksum}" "sha512sum" assert_success - assert_output --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test" + assert_line --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test" rm -rf "${CONTAINERBASE_CACHE_DIR}" @@ -236,6 +236,7 @@ teardown() { } @test "get_from_url_with_cache_and_checksum" { + bats_require_minimum_version 1.5.0 # create cache dir CONTAINERBASE_CACHE_DIR="${TEST_ROOT_DIR}/cache" mkdir -p "${CONTAINERBASE_CACHE_DIR}" @@ -244,7 +245,7 @@ teardown() { local checksum="233c335a7f10e9f0dfd7e9d0cda802a38c15a7f13b6678c55980814f22799a70590d56888a819b6591881ec1939240d9dbe68e7e495021b4d6c6a49cdee24d80" local file="https://github.com/containerbase/base/releases/download/7.10.0/containerbase.tar.xz" - run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha512sum" + run --separate-stderr get_from_url "${file}" $(basename "${file}") "${checksum}" "sha512sum" assert_success assert_output --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz" @@ -252,7 +253,7 @@ teardown() { run get_from_url "${file}" test "${checksum}" "sha512sum" assert_success - assert_output --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test" + assert_line --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test" # change checksum of cached file echo "a" >> "${file_path}" diff --git a/test/bash/util.sh b/test/bash/util.sh index b8110d0f9..20e596ab7 100644 --- a/test/bash/util.sh +++ b/test/bash/util.sh @@ -19,3 +19,15 @@ function is_root () { echo 0 fi } + +function link_cli_tool () { + local arch=x64 + + if [[ "${ARCHITECTURE}" = "aarch64" ]];then + arch=arm64 + fi + mkdir -p "${BIN_DIR}" + export PATH="${BIN_DIR}:${PATH}" + ln -sf "${CONTAINERBASE_DIR}/bin/containerbase-cli-${arch}" "${BIN_DIR}/containerbase-cli" +} +link_cli_tool