From 12a9431c414d4557d068e2884124715995bd88f8 Mon Sep 17 00:00:00 2001 From: oren-codefresh Date: Tue, 23 Jun 2020 17:59:11 +0300 Subject: [PATCH] Saas 7579 (#514) Download components on image to offline --- Dockerfile | 1 + lib/binary/downloader.js | 43 +++++----- lib/binary/index.js | 3 +- .../cli/commands/cluster/cluster.sdk.spec.js | 9 +-- .../cli/commands/cluster/create.cmd.js | 49 ++++++----- .../cli/commands/components/update.cmd.js | 25 ++++++ lib/interface/cli/commands/hybrid/helper.js | 71 +++++++++++++--- lib/interface/cli/commands/hybrid/init.cmd.js | 3 +- .../cli/commands/monitor/install.cmd.js | 81 +++++++++++-------- .../cli/commands/root/components.cmd.js | 17 ++++ package.json | 2 +- 11 files changed, 202 insertions(+), 102 deletions(-) create mode 100644 lib/interface/cli/commands/components/update.cmd.js create mode 100644 lib/interface/cli/commands/root/components.cmd.js diff --git a/Dockerfile b/Dockerfile index 95daad58e..1e817fd20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,4 +39,5 @@ RUN apk del yarn RUN ln -s $(pwd)/lib/interface/cli/codefresh /usr/local/bin/codefresh +RUN codefresh components update --location components ENTRYPOINT ["codefresh"] diff --git a/lib/binary/downloader.js b/lib/binary/downloader.js index 5e4b8c3c1..46cbbc2db 100644 --- a/lib/binary/downloader.js +++ b/lib/binary/downloader.js @@ -28,28 +28,16 @@ function _ensureDirectory(location) { } async function _getRemoteVersion({ - name, osType, + name, branch, path, file, }) { - // get the 10 latest releases - const releasesUrl = `https://api.github.com/repos/codefresh-io/${name}/releases?per_page=10`; - const res = await rp({ - uri: releasesUrl, + const final = branch ? `${name}/${branch}/${path}` : `${name}/${path}`; + const url = `https://raw.githubusercontent.com/codefresh-io/${final}/${file}`; + const req = await rp({ + url, method: 'GET', headers: { 'User-Agent': 'codefresh' }, }); - - const releases = JSON.parse(res); - for (let i = 0; i < releases.length; i += 1) { - const curRelease = releases[i]; - const neededAsset = (curRelease.assets || []).find(asset => { - const assetName = asset.name || ''; - return assetName.startsWith(`${name}_`) && assetName.endsWith(osType); - }); - if (neededAsset) { - return { remoteVersion: curRelease.name, download_url: neededAsset.browser_download_url }; - } - } - return { remoteVersion: '0', download_url: '' }; + return req; } function _buildDownloadURL({ name, version, binary }) { @@ -118,19 +106,26 @@ class Downloader { let localVersion = _getLocalVersion(join(dir, component.local.versionFile)); const { - repo: name, + repo: name, branch, versionPath: path, versionFile: file, } = component.remote; - let { remoteVersion, download_url } = await _getRemoteVersion({ name, osType }); + let remoteVersion = await _getRemoteVersion({ + name, branch, path, file, + }); remoteVersion = remoteVersion.trim(); localVersion = localVersion.trim(); if (compareVersions(localVersion, remoteVersion) >= 0) { + // logger.debug(`Download is not required latest-version=${remoteVersion} local-version=${localVersion}`); return Promise.resolve(); } - logger.debug(`${component.name} component upgrade is required, downloading latest version [${remoteVersion}]`); + logger.debug(`${component.name} component upgrade is required, downloading.`); - const resp = await request(download_url); + const binary = `${name}_${remoteVersion}_${osType}`; + const version = component.version.prefix ? `${component.version.prefix}${remoteVersion}` : remoteVersion; + const url = _buildDownloadURL({ name, version, binary }); + const resp = await request(url); + if (this.progress) { let size = 0; resp.on('response', (res) => { @@ -161,6 +156,9 @@ class Downloader { } resolveFn(); }); + resp.on('error', (err) => { + rejectFn(err); + }); }); } } @@ -168,4 +166,5 @@ class Downloader { module.exports = { CommonProgressFormat: 'downloading [{bar}] {percentage}% | {value}/{total}', Downloader, + CODEFRESH_PATH, }; diff --git a/lib/binary/index.js b/lib/binary/index.js index a1edeefa8..79542f4a0 100644 --- a/lib/binary/index.js +++ b/lib/binary/index.js @@ -1,10 +1,11 @@ const components = require('./components'); const { Runner } = require('./runner'); -const { Downloader, CommonProgressFormat } = require('./downloader'); +const { Downloader, CommonProgressFormat, CODEFRESH_PATH } = require('./downloader'); module.exports = { components, Runner, Downloader, CommonProgressFormat, + CODEFRESH_PATH, }; diff --git a/lib/interface/cli/commands/cluster/cluster.sdk.spec.js b/lib/interface/cli/commands/cluster/cluster.sdk.spec.js index 1639ed2bf..777ac44bd 100644 --- a/lib/interface/cli/commands/cluster/cluster.sdk.spec.js +++ b/lib/interface/cli/commands/cluster/cluster.sdk.spec.js @@ -1,12 +1,12 @@ const getCmd = require('./get.cmd').toCommand(); const deleteCmd = require('./delete.cmd').toCommand(); const createCmd = require('./create.cmd').toCommand(); -const { sdk } = require('../../../../logic'); - +const { Runner } = require('./../../../../binary'); const request = require('requestretry'); jest.mock('../../../../logic/entities/Cluster'); - +jest.mock('../hybrid/helper'); +jest.mock('./../../../../binary'); const DEFAULT_RESPONSE = request.__defaultResponse(); @@ -29,10 +29,9 @@ describe('cluster commands', () => { describe('create', () => { it('should skip', async () => { - jest.spyOn(sdk.clusters, 'create').mockImplementation(); const argv = {}; await createCmd.handler(argv); - expect(sdk.clusters.create).toBeCalled(); + expect(Runner.mock.instances[0].run).toBeCalled(); }); }); diff --git a/lib/interface/cli/commands/cluster/create.cmd.js b/lib/interface/cli/commands/cluster/create.cmd.js index 890e44034..06f95e914 100644 --- a/lib/interface/cli/commands/cluster/create.cmd.js +++ b/lib/interface/cli/commands/cluster/create.cmd.js @@ -1,8 +1,8 @@ const Command = require('../../Command'); const createRoot = require('../root/create.cmd'); const { sdk } = require('../../../../logic'); -const ProgressEvents = require('../../helpers/progressEvents'); -const cliProgress = require('cli-progress'); +const { downloadSteveDore } = require('../hybrid/helper'); +const { Runner, components } = require('../../../../binary'); const command = new Command({ command: 'clusters [name]', @@ -53,33 +53,30 @@ const command = new Command({ if (terminateProcess === undefined) { terminateProcess = true; } - const events = new ProgressEvents(); - const format = 'downloading cluster installer [{bar}] {percentage}% | {value}/{total}'; - const progressBar = new cliProgress.SingleBar( - { stopOnComplete: true, format }, - cliProgress.Presets.shades_classic, - ); - let clusterTotalSize; - events.onStart((size) => { - progressBar.start(size, 0); - clusterTotalSize = size; - }); - events.onProgress((progress) => { - progressBar.update(progress); - if (progress >= clusterTotalSize) { - console.log('\n'); - } - }); - return sdk.clusters.create({ + const binLocation = await downloadSteveDore(); + const componentRunner = new Runner(binLocation); + const commands = [ + 'create', + '--c', contextName, - context, + '--token', + context.token, + '--api-host', + `${context.url}/`, + '--namespace', namespace, + '--serviceaccount', serviceaccount, - behindFirewall, - name, - terminateProcess, - events, - }); + ]; + if (name) { + commands.push('--name-overwrite'); + commands.push(name); + } + if (behindFirewall) { + commands.push('--behind-firewall'); + } + + await componentRunner.run(components.stevedore, commands); }, }); diff --git a/lib/interface/cli/commands/components/update.cmd.js b/lib/interface/cli/commands/components/update.cmd.js new file mode 100644 index 000000000..1539c0145 --- /dev/null +++ b/lib/interface/cli/commands/components/update.cmd.js @@ -0,0 +1,25 @@ +const Command = require('../../Command'); +const componentsRoot = require('../root/components.cmd'); +const helper = require('../hybrid/helper'); + +const command = new Command({ + command: 'update', + parent: componentsRoot, + description: 'Update Codefresh CLI components', + webDocs: { + category: 'Componenets', + title: 'Update', + }, + builder: yargs => yargs + .env('CF_ARG_') // this means that every process.env.CF_ARG_* will be passed to argv + .option('location', { + describe: 'Override dowload folder location', + }), + handler: async (argv) => { + console.log('Updating components'); + const { location } = argv; + await helper.downloadRelatedComponents(location); + }, +}); + +module.exports = command; diff --git a/lib/interface/cli/commands/hybrid/helper.js b/lib/interface/cli/commands/hybrid/helper.js index 1b112eefe..b70699631 100644 --- a/lib/interface/cli/commands/hybrid/helper.js +++ b/lib/interface/cli/commands/hybrid/helper.js @@ -10,9 +10,12 @@ const { followLogs } = require('../../helpers/logs'); const ProgressEvents = require('../../helpers/progressEvents'); const cliProgress = require('cli-progress'); const figlet = require('figlet'); +const path = require('path'); const { components, Runner, Downloader, CommonProgressFormat, + CODEFRESH_PATH, } = require('./../../../../binary'); +const { pathExists } = require('../../helpers/general'); const INSTALLATION_DEFAULTS = { NAMESPACE: 'codefresh', @@ -23,6 +26,7 @@ const INSTALLATION_DEFAULTS = { CF_CONTEXT_NAME: 'cf-runner', STORAGE_CLASS_PREFIX: 'dind-local-volumes-runner', RESUME_OLD_INSTALLATION: true, + COMPONENTS_FOLDER: 'components', }; const maxRuntimeNameLength = 63; @@ -308,7 +312,7 @@ async function getRecommendedKubeNamespace(kubeconfigPath, kubeContextName) { return name; } -async function _downloadVeonona() { +async function downloadVeonona(location = CODEFRESH_PATH) { const downloader = new Downloader({ progress: new cliProgress.SingleBar( { @@ -317,13 +321,57 @@ async function _downloadVeonona() { }, cliProgress.Presets.shades_classic, ), + location, }); - await downloader.download(components.venona); + const [error] = await to(downloader.download(components.venona)); + if (error) { + const newLocation = path.join(INSTALLATION_DEFAULTS.COMPONENTS_FOLDER, components.venona.local.dir, components.venona.local.binary); + if (await pathExists(newLocation)) { + console.log('Failed to download installer, using binary from components folder'); + return path.resolve(process.cwd(), INSTALLATION_DEFAULTS.COMPONENTS_FOLDER); + } + console.log('Failed to download component, aborting'); + throw error; + } + return location; +} +async function downloadSteveDore(location = CODEFRESH_PATH) { + const downloader = new Downloader({ + progress: new cliProgress.SingleBar( + { + stopOnComplete: true, + format: CommonProgressFormat, + }, + cliProgress.Presets.shades_classic, + ), + location, + }); + const [error] = await to(downloader.download(components.stevedore)); + if (error) { + const newLocation = path.join( + INSTALLATION_DEFAULTS.COMPONENTS_FOLDER, + components.stevedore.local.dir, components.stevedore.local.binary, + ); + if (await pathExists(newLocation)) { + console.log('Failed to download installer, using binary from components folder'); + return path.resolve(process.cwd(), INSTALLATION_DEFAULTS.COMPONENTS_FOLDER); + } + console.log('Failed to download component, aborting'); + throw error; + } + return location; +} + +async function downloadHybridComponents(location) { + await downloadVeonona(location); + console.log(`Kubernetes components installer downloaded successfully to ${location} `); + await downloadSteveDore(location); + console.log(`Kubernetes registrator installer downloaded successfully ${location}`); } async function runClusterAcceptanceTests({ kubeNamespace, kubeConfigPath }) { - await _downloadVeonona(); - const componentRunner = new Runner(); + const binLocation = await downloadVeonona(); + const componentRunner = new Runner(binLocation); const cmd = ['test', '--log-formtter', DefaultLogFormatter]; if (kubeNamespace) { cmd.push('--kube-namespace'); @@ -353,8 +401,8 @@ async function installAgent({ verbose, // --verbose logFormatting = DefaultLogFormatter, // --log-formtter }) { - await _downloadVeonona(); - const componentRunner = new Runner(); + const binLocation = await downloadVeonona(); + const componentRunner = new Runner(binLocation); const cmd = [ 'install', 'agent', @@ -422,8 +470,8 @@ async function installRuntime({ storageClassName, // --storage-class logFormatting = DefaultLogFormatter, // --log-formtter }) { - await _downloadVeonona(); - const componentRunner = new Runner(); + const binLocation = await downloadVeonona(); + const componentRunner = new Runner(binLocation); const cmd = [ 'install', 'runtime', @@ -487,8 +535,8 @@ async function attachRuntime({ runtimeName, // --runtimeName logFormatting = DefaultLogFormatter, // --log-formtter }) { - await _downloadVeonona(); - const componentRunner = new Runner(); + const binLocation = await downloadVeonona(); + const componentRunner = new Runner(binLocation); const cmd = [ 'attach', '--kube-context-name', @@ -615,6 +663,9 @@ module.exports = { newRuntimeName, newAgentName, parseNodeSelector, + downloadRelatedComponents: downloadHybridComponents, + downloadSteveDore, + downloadVeonona, INSTALLATION_DEFAULTS, DefaultLogFormatter, }; diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js index 8dc183eae..43f8dd76e 100644 --- a/lib/interface/cli/commands/hybrid/init.cmd.js +++ b/lib/interface/cli/commands/hybrid/init.cmd.js @@ -111,7 +111,7 @@ const initCmd = new Command({ .option('install-monitor', { describe: 'Install a monitoring component that will help provide valueable data about your cluster to Codefresh', type: 'boolean', - default: true, + default: true, }) .option('kube-namespace', { describe: 'Name of the namespace on which venona should be installed [$CF_ARG_KUBE_NAMESPACE]', @@ -330,7 +330,6 @@ const initCmd = new Command({ installationPlan.addContext('agent', agent); const { token: agentToken } = agent; console.log(`A Codefresh Runner with the name: ${colors.cyan(agentName)} has been created.`); - console.log(agentToken); installationPlan.addContext('agentToken', agentToken); }, condition: async () => !installationPlan.getContext('agentToken'), diff --git a/lib/interface/cli/commands/monitor/install.cmd.js b/lib/interface/cli/commands/monitor/install.cmd.js index 94269d719..10152be2a 100644 --- a/lib/interface/cli/commands/monitor/install.cmd.js +++ b/lib/interface/cli/commands/monitor/install.cmd.js @@ -2,9 +2,9 @@ const Command = require('../../Command'); const installRoot = require('../root/install.cmd'); const { sdk } = require('../../../../logic'); -const ProgressEvents = require('../../helpers/progressEvents'); -const cliProgress = require('cli-progress'); const { DefaultLogFormatter } = require('./../hybrid/helper'); +const { downloadVeonona } = require('../hybrid/helper'); +const { Runner, components } = require('../../../../binary'); const installMonitorCmd = new Command({ root: false, @@ -39,49 +39,60 @@ const installMonitorCmd = new Command({ handler: async (argv) => { const { url, - 'kube-config-path': kubeConfigPath, + // 'kube-config-path': kubeConfigPath, 'cluster-id': clusterId, token, 'kube-context-name': kubeContextName, 'kube-namespace': kubeNamespace, verbose, - noExit, + // noExit, } = argv; const apiHost = sdk.config.context.url; - const events = new ProgressEvents(); - const format = 'downloading [{bar}] {percentage}% | {value}/{total}'; - const progressBar = new cliProgress.SingleBar({ stopOnComplete: true, format }, cliProgress.Presets.shades_classic); - let totalSize; - events.onStart((size) => { - console.log('Downloading agent\'s installer \n'); - progressBar.start(size, 0); - totalSize = size; - }); - events.onProgress((progress) => { - progressBar.update(progress); - if (progress >= totalSize) { - console.log('\n'); - } - }); - const monitorInstallStatusCode = await sdk.monitor.install({ - apiHost, - kubeContextName, - kubeNamespace, - token, - clusterId, - kubeConfigPath, - verbose, - events, - codefreshHost: url, - logFormatting: DefaultLogFormatter, - }); - if (monitorInstallStatusCode !== 0) { - throw new Error(`\nCodefresh Monitoring installation failed with code ${monitorInstallStatusCode}`); + + const binLocation = await downloadVeonona(); + const componentRunner = new Runner(binLocation); + + const commands = [ + 'install', + 'monitor', + ]; + + if (clusterId) { + commands.push('--clusterId'); + commands.push(clusterId); + } + + if (token) { + commands.push('--codefreshToken'); + commands.push(token); } - if (!noExit) { - process.exit(0); + if (kubeContextName) { + commands.push('--kube-context-name'); + commands.push(kubeContextName); } + + // if (helm3) { + // commands.push('--helm3'); + // } + + if (kubeNamespace) { + commands.push('--kube-namespace'); + commands.push(kubeNamespace); + } + + if (url) { + commands.push('--codefreshHost'); + commands.push(url); + } + if (verbose) { + commands.push('--verbose'); + } + if (DefaultLogFormatter) { + commands.push(`--log-formtter=${DefaultLogFormatter}`); + } + + await componentRunner.run(components.venona, commands); }, }); diff --git a/lib/interface/cli/commands/root/components.cmd.js b/lib/interface/cli/commands/root/components.cmd.js new file mode 100644 index 000000000..17db9c9d3 --- /dev/null +++ b/lib/interface/cli/commands/root/components.cmd.js @@ -0,0 +1,17 @@ +const Command = require('../../Command'); + + +const install = new Command({ + root: true, + command: 'components', + description: 'Manages Codefresh CLI components', + requiresAuthentication: false, + webDocs: { + description: 'Codefresh CLI components', + category: 'Codefresh CLI components', + title: 'components', + weight: 40, + }, +}); + +module.exports = install; diff --git a/package.json b/package.json index 895667702..020af4e34 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.68.17", + "version": "0.69.0", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true,