feat: support url rewrite (#1210)

containerbase · Jul 22, 2023 · cdec808 · cdec808
1 parent 45e1e2a
commit cdec808
Show file tree

Hide file tree

Showing 8 changed files with 88 additions and 14 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -86,6 +86,9 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
+      - name: build
+        run: yarn build
+
       - name: bats
         run: yarn test:bats
 

diff --git a/README.md b/README.md
@@ -98,10 +98,9 @@ You can replace the default urls used to download the tools.
 The number of `URL_REPLACE_*_FROM` and `URL_REPLACE_*_TO` environment variables must match.
 Theses variables are case sensitive.
 The numbers will be processed in numerical order and can have gaps.
-This is currently only supported by these tool installers:
-
-- `docker`
-- `dart`
+Almost all tools are now supported.
+Only tools which are installed by `gem`, `npm` or `pip` are not supported.
+They can be configured via thier own environment variables.
 
 Checkout [#1067](https://github.com/containerbase/base/issues/1067) for additional support.
 

diff --git a/src/cli/command/download-file.ts b/src/cli/command/download-file.ts
@@ -0,0 +1,54 @@
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { Command, Option } from 'clipanion';
+import { got } from 'got';
+import prettyMilliseconds from 'pretty-ms';
+import { EnvService, rootContainer } from '../services';
+import { logger } from '../utils';
+
+export class DownloadFileCommand extends Command {
+  static override paths = [['download', 'file'], ['df']];
+
+  static override usage = Command.Usage({
+    description: 'Downloads a file and optionally validates the checksum.',
+  });
+
+  url = Option.String({ required: true });
+  output = Option.String({ required: true });
+
+  // checksum = Option.String('-c,--checksum');
+  // algo = Option.String('-a,--algo');
+
+  // dryRun = Option.Boolean('-d,--dry-run', false);
+
+  async execute(): Promise<number | void> {
+    const start = Date.now();
+    let error = false;
+    logger.info({ url: this.url, output: this.output }, `Downloading file ...`);
+    try {
+      const container = rootContainer.createChild();
+
+      const env = container.get(EnvService);
+      const path = dirname(this.output);
+
+      await mkdir(path, { recursive: true });
+
+      const nUrl = env.replaceUrl(this.url);
+      await pipeline(got.stream(nUrl), createWriteStream(this.output));
+
+      return 0;
+    } catch (err) {
+      logger.fatal(err);
+      error = true;
+      return 1;
+    } finally {
+      logger.info(
+        `Download completed ${
+          error ? 'with errors ' : ''
+        } in ${prettyMilliseconds(Date.now() - start)}.`
+      );
+    }
+  }
+}
diff --git a/src/cli/command/index.ts b/src/cli/command/index.ts
@@ -2,6 +2,7 @@ import { argv0 } from 'node:process';
 import type { Cli } from 'clipanion';
 import type { CliMode } from '../utils';
 import { logger } from '../utils/logger';
+import { DownloadFileCommand } from './download-file';
 import { InstallToolCommand, InstallToolShortCommand } from './install-tool';
 import { PrepareToolCommand, PrepareToolShortCommand } from './prepare-tool';
 import { prepareToolVersion } from './utils';
@@ -28,6 +29,7 @@ export function prepareCommands(
 
   cli.register(InstallToolCommand);
   cli.register(PrepareToolCommand);
+  cli.register(DownloadFileCommand);
   return prepareToolVersion(mode, args);
 }
 

diff --git a/src/usr/local/containerbase/util.sh b/src/usr/local/containerbase/util.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# https://github.com/vercel/pkg/issues/1861
+unset PKG_EXECPATH
+
 # get path location
 DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi

diff --git a/src/usr/local/containerbase/utils/cache.sh b/src/usr/local/containerbase/utils/cache.sh
@@ -90,7 +90,7 @@ function download_file () {
   local temp_folder=${CONTAINERBASE_CACHE_DIR:-${TEMP_DIR}}
   while [ "${retry}" -gt 0 ]; do
     retry=$((retry-1))
-    if ! curl --retry 3 --create-dirs -sSfLo "${temp_folder}/${name}" "${url}" ; then
+    if ! containerbase-cli df "${url}" "${temp_folder}/${name}" >&2; then
       echo "Download failed: ${url}" >&2
       exit 1
     fi;
@@ -119,7 +119,7 @@ function download_file () {
 # First argument is the path to the file
 # Second argument is the checksum
 # Third argument is the type, currently supported:
-# * sha1
+# * sha1sum
 # * sha224sum
 # * sha256sum
 # * sha384sum

diff --git a/test/bash/cache.bats b/test/bash/cache.bats
@@ -156,11 +156,11 @@ teardown() {
 
   run download_file "${file}"
   assert_success
-  assert_output "${CONTAINERBASE_CACHE_DIR}/containerbase.tar.xz"
+  assert_line "${CONTAINERBASE_CACHE_DIR}/containerbase.tar.xz"
 
   run download_file "${file}" "foobar"
   assert_success
-  assert_output "${CONTAINERBASE_CACHE_DIR}/foobar"
+  assert_line "${CONTAINERBASE_CACHE_DIR}/foobar"
 
   CONTAINERBASE_CACHE_DIR= \
   tmp_file=$(download_file "${file}")
@@ -178,11 +178,11 @@ teardown() {
 
   run get_from_url "${file}"
   assert_success
-  assert_output --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz"
+  assert_line --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz"
 
   run get_from_url "${file}" test
   assert_success
-  assert_output --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test"
+  assert_line --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test"
 
   # overwrite donwload function to fail
   function download_file () {
@@ -209,13 +209,13 @@ teardown() {
 
   run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha512sum"
   assert_success
-  assert_output --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz"
+  assert_line --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz"
 
   rm -rf "${CONTAINERBASE_CACHE_DIR}"
 
   run get_from_url "${file}" test "${checksum}" "sha512sum"
   assert_success
-  assert_output --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test"
+  assert_line --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test"
 
   rm -rf "${CONTAINERBASE_CACHE_DIR}"
 
@@ -236,6 +236,7 @@ teardown() {
 }
 
 @test "get_from_url_with_cache_and_checksum" {
+  bats_require_minimum_version 1.5.0
   # create cache dir
   CONTAINERBASE_CACHE_DIR="${TEST_ROOT_DIR}/cache"
   mkdir -p "${CONTAINERBASE_CACHE_DIR}"
@@ -244,15 +245,15 @@ teardown() {
   local checksum="233c335a7f10e9f0dfd7e9d0cda802a38c15a7f13b6678c55980814f22799a70590d56888a819b6591881ec1939240d9dbe68e7e495021b4d6c6a49cdee24d80"
   local file="https://github.com/containerbase/base/releases/download/7.10.0/containerbase.tar.xz"
 
-  run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha512sum"
+  run --separate-stderr get_from_url "${file}" $(basename "${file}") "${checksum}" "sha512sum"
   assert_success
   assert_output --regexp "^${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/containerbase\.tar\.xz"
 
   file_path="${output}"
 
   run get_from_url "${file}" test "${checksum}" "sha512sum"
   assert_success
-  assert_output --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test"
+  assert_line --regexp "${CONTAINERBASE_CACHE_DIR}/[0-9a-f]{64}/test"
 
   # change checksum of cached file
   echo "a" >> "${file_path}"

diff --git a/test/bash/util.sh b/test/bash/util.sh
@@ -19,3 +19,15 @@ function is_root () {
     echo 0
   fi
 }
+
+function link_cli_tool () {
+  local arch=x64
+
+  if [[ "${ARCHITECTURE}" = "aarch64" ]];then
+    arch=arm64
+  fi
+  mkdir -p "${BIN_DIR}"
+  export PATH="${BIN_DIR}:${PATH}"
+  ln -sf "${CONTAINERBASE_DIR}/bin/containerbase-cli-${arch}" "${BIN_DIR}/containerbase-cli"
+}
+link_cli_tool