From d592c092abeeca17c56b87df6c0ec3b6011de857 Mon Sep 17 00:00:00 2001 From: Sergio Cinos Date: Tue, 10 Nov 2020 11:11:02 +0100 Subject: [PATCH 1/4] Modernize launch options for VSCode --- .vscode/launch.json | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index b108660..30764b4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,24 +3,34 @@ // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", - "configurations": [ + "configurations": [ { - "type": "node", - "request": "attach", - "name": "Attach", - "port": 9229 - }, - { - "type": "node", "request": "launch", "name": "Debug", - "program": "${workspaceRoot}/build/index.js", - "smartStep": true, + "program": "${workspaceFolder}/build/index.js", + "skipFiles": [ + "/**" + ], "outFiles": [ - "../dist/**/*.js" + "${workspaceFolder}/build/**/*.js", + "!**/node_modules/**" ], "preLaunchTask": "npm: build-ts", - "protocol": "inspector" + "type": "pwa-node", + "outputCapture": "std" + }, + { + "request": "launch", + "name": "Test", + "program": "${workspaceFolder}/node_modules/.bin/jest", + "skipFiles": [ + "/**" + ], + "args": [ + "--runInBand", + "${file}" + ], + "type": "pwa-node", } ] } From 6da84c0cd8295cd410ab4bd1d37a9a180d01f060 Mon Sep 17 00:00:00 2001 From: Sergio Cinos Date: Tue, 10 Nov 2020 11:21:18 +0100 Subject: [PATCH 2/4] Adds esModuleInterop to tsconfig --- src/api.ts | 14 +++++++------- src/app/app-shell.tsx | 2 +- src/app/debug.tsx | 8 ++++---- src/app/index.tsx | 4 ++-- src/app/local-images.tsx | 6 +++--- src/app/log-details.tsx | 2 +- src/app/log.tsx | 4 ++-- src/builder.ts | 10 +++++----- src/config.ts | 2 +- src/daemon.ts | 2 +- src/index.ts | 8 ++++---- src/logger.ts | 4 ++-- src/middlewares.ts | 4 ++-- tsconfig.json | 3 ++- 14 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/api.ts b/src/api.ts index cee68be..75b5058 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,10 +1,10 @@ -import * as httpProxy from 'http-proxy'; -import * as Docker from 'dockerode'; -import * as _ from 'lodash'; -import * as portfinder from 'portfinder'; -import * as git from 'nodegit'; -import * as fs from 'fs-extra'; -import * as path from 'path'; +import httpProxy from 'http-proxy'; +import Docker from 'dockerode'; +import _ from 'lodash'; +import portfinder from 'portfinder'; +import git from 'nodegit'; +import fs from 'fs-extra'; +import path from 'path'; import { promisify } from 'util'; import { ContainerInfo } from 'dockerode'; diff --git a/src/app/app-shell.tsx b/src/app/app-shell.tsx index 056c745..6def9a3 100644 --- a/src/app/app-shell.tsx +++ b/src/app/app-shell.tsx @@ -1,4 +1,4 @@ -import * as React from 'react'; +import React from 'react'; import { start } from 'repl'; import { humanRelativeTime } from './util'; import { pendingHashes, buildQueue } from '../builder'; diff --git a/src/app/debug.tsx b/src/app/debug.tsx index ec77c7f..48ed9c9 100644 --- a/src/app/debug.tsx +++ b/src/app/debug.tsx @@ -1,7 +1,7 @@ -import * as React from 'react'; -import * as ReactDOMServer from 'react-dom/server'; -import * as Dockerode from 'dockerode'; -import * as os from 'os'; +import React from 'react'; +import ReactDOMServer from 'react-dom/server'; +import Dockerode from 'dockerode'; +import os from 'os'; import { ONE_MINUTE } from '../constants'; import { Shell } from './app-shell'; diff --git a/src/app/index.tsx b/src/app/index.tsx index abf4ee8..fceef49 100644 --- a/src/app/index.tsx +++ b/src/app/index.tsx @@ -1,5 +1,5 @@ -import * as React from 'react'; -import * as ReactDOMServer from 'react-dom/server'; +import React from 'react'; +import ReactDOMServer from 'react-dom/server'; import { ONE_SECOND } from '../constants'; import { Shell } from './app-shell'; diff --git a/src/app/local-images.tsx b/src/app/local-images.tsx index cb27949..4947c4a 100644 --- a/src/app/local-images.tsx +++ b/src/app/local-images.tsx @@ -1,6 +1,6 @@ -import * as React from 'react'; -import * as ReactDOMServer from 'react-dom/server'; -import * as Docker from 'dockerode'; +import React from 'react'; +import ReactDOMServer from 'react-dom/server'; +import Docker from 'dockerode'; import { config } from '../config'; import { Shell } from './app-shell'; diff --git a/src/app/log-details.tsx b/src/app/log-details.tsx index 0e1cab2..754a128 100644 --- a/src/app/log-details.tsx +++ b/src/app/log-details.tsx @@ -1,4 +1,4 @@ -import * as React from 'react'; +import React from 'react'; import { humanTimeSpan } from './util'; const interestingDetails = new Set( [ diff --git a/src/app/log.tsx b/src/app/log.tsx index fe0e726..5375a45 100644 --- a/src/app/log.tsx +++ b/src/app/log.tsx @@ -1,5 +1,5 @@ -import * as React from 'react'; -import * as ReactDOMServer from 'react-dom/server'; +import React from 'react'; +import ReactDOMServer from 'react-dom/server'; import { Shell } from './app-shell'; import { errorClass, humanRelativeTime } from './util'; diff --git a/src/builder.ts b/src/builder.ts index d29ee15..18843d8 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -1,8 +1,8 @@ -import * as fs from 'fs-extra'; -import * as git from 'nodegit'; -import * as os from 'os'; -import * as path from 'path'; -import * as tar from 'tar-fs'; +import fs from 'fs-extra'; +import git from 'nodegit'; +import os from 'os'; +import path from 'path'; +import tar from 'tar-fs'; import { Readable } from 'stream'; import { sample } from 'lodash'; diff --git a/src/config.ts b/src/config.ts index f6baa77..0e6ad9d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import * as Dockerode from 'dockerode'; +import Dockerode from 'dockerode'; import { RunEnv } from './api'; type Readonly< T > = { readonly [ P in keyof T ]: T[ P ] }; diff --git a/src/daemon.ts b/src/daemon.ts index d53f49c..393a836 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,4 +1,4 @@ -import * as forever from 'forever-monitor'; +import forever from 'forever-monitor'; import { l } from './logger'; diff --git a/src/index.ts b/src/index.ts index d186a16..2000a33 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,8 @@ // external -import * as express from 'express'; -import * as fs from 'fs-extra'; -import * as striptags from 'striptags'; -import * as useragent from 'useragent'; +import express from 'express'; +import fs from 'fs-extra'; +import striptags from 'striptags'; +import useragent from 'useragent'; import { exec } from 'child_process'; // internal diff --git a/src/logger.ts b/src/logger.ts index c995afc..771f62c 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,5 +1,5 @@ -import * as bunyan from 'bunyan'; -import * as _ from 'lodash'; +import bunyan from 'bunyan'; +import _ from 'lodash'; import { Writable } from 'stream'; import { CommitHash, getImageName } from './api'; diff --git a/src/middlewares.ts b/src/middlewares.ts index d54a3ba..d8924b4 100644 --- a/src/middlewares.ts +++ b/src/middlewares.ts @@ -1,6 +1,6 @@ // external -import * as express from 'express'; -import * as expressSession from 'express-session'; +import express from 'express'; +import expressSession from 'express-session'; // internal import { getCommitHashForBranch, refreshRemoteBranches, CommitHash, touchCommit, RunEnv } from './api'; diff --git a/tsconfig.json b/tsconfig.json index 35ad56b..ac1d86c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,7 +7,8 @@ "sourceMap": true, "outDir": "build", "jsx": "react", - "lib": [ "es7", "dom" ] + "lib": [ "es7", "dom" ], + "esModuleInterop": true }, "include": [ "src/*" ] } From 7e79f650e9e3971fd81e3746377f2cadd204444d Mon Sep 17 00:00:00 2001 From: Sergio Cinos Date: Tue, 10 Nov 2020 11:29:02 +0100 Subject: [PATCH 3/4] Prettify files --- src/api.ts | 17 ++++++++++++----- src/builder.ts | 16 ++++++++++------ src/config.ts | 6 +++--- src/middlewares.ts | 30 ++++++++++++++++++++++++------ 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/api.ts b/src/api.ts index cee68be..4989122 100644 --- a/src/api.ts +++ b/src/api.ts @@ -55,7 +55,7 @@ export const extractCommitFromImage = ( imageName: string ): CommitHash => { export const extractEnvironmentFromImage = ( image: ContainerInfo ): RunEnv => { return image.Labels.calypsoEnvironment || undefined; -} +}; /** * Polls the local Docker daemon to @@ -138,7 +138,11 @@ export async function startContainer( commitHash: CommitHash, env: RunEnv ) { return state.startingContainers.get( containerId ); } - async function start( image: string, commitHash: CommitHash, env: RunEnv ): Promise< ContainerInfo > { + async function start( + image: string, + commitHash: CommitHash, + env: RunEnv + ): Promise< ContainerInfo > { // ok, try to start one let freePort: number; try { @@ -230,7 +234,10 @@ export async function refreshRunningContainers() { export function getRunningContainerForHash( hash: CommitHash, env?: RunEnv ): ContainerInfo | null { const image = getImageName( hash ); return Array.from( state.containers.values() ).find( - ci => ci.Image === image && ci.State === 'running' && ( ! env || env === extractEnvironmentFromImage( ci ) ) + ci => + ci.Image === image && + ci.State === 'running' && + ( ! env || env === extractEnvironmentFromImage( ci ) ) ); } @@ -296,7 +303,7 @@ async function getRemoteBranches(): Promise< Map< string, string > > { timing( 'git.refresh', Date.now() - start ); try { - const branchesReferences = (await repo.getReferences( )).filter( + const branchesReferences = ( await repo.getReferences() ).filter( ( x: git.Reference ) => x.isBranch ); @@ -468,7 +475,7 @@ export async function proxyRequestToHash( req: any, res: any ) { } proxy.web( req, res, { target: `http://localhost:${ port }` }, err => { - if ( err && (err as any).code === "ECONNRESET") { + if ( err && ( err as any ).code === 'ECONNRESET' ) { return; } l.log( { err, req, res, commitHash }, 'unexpected error occured while proxying' ); diff --git a/src/builder.ts b/src/builder.ts index d29ee15..d4b4dfe 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -27,10 +27,11 @@ import { increment, timing, gauge } from './stats'; export const MAX_CONCURRENT_BUILDS = 4; const MAX_CORES = [ 12 ]; -export const getBuildConcurrency = () => Math.max( - 1, - Math.min( Math.floor( os.cpus().length / MAX_CONCURRENT_BUILDS ), sample( MAX_CORES ) ) -); +export const getBuildConcurrency = () => + Math.max( + 1, + Math.min( Math.floor( os.cpus().length / MAX_CONCURRENT_BUILDS ), sample( MAX_CORES ) ) + ); export const buildQueue: Array< CommitHash > = []; export const pendingHashes: Set< CommitHash > = new Set(); @@ -147,7 +148,10 @@ export async function buildImageForHash( commitHash: CommitHash ): Promise< void const buildConcurrency = getBuildConcurrency(); try { - l.log( { commitHash, buildDir, repoDir, imageName, buildConcurrency }, 'Attempting to build image.' ); + l.log( + { commitHash, buildDir, repoDir, imageName, buildConcurrency }, + 'Attempting to build image.' + ); const cloneStart = Date.now(); buildLogger.info( 'Cloning git repo' ); @@ -212,7 +216,7 @@ export async function buildImageForHash( commitHash: CommitHash ): Promise< void if ( ! err ) { const buildImageTime = Date.now() - imageStart; timing( 'build_image', buildImageTime ); - timing( `build_image_by_core.${buildConcurrency}_cores`, buildImageTime ); + timing( `build_image_by_core.${ buildConcurrency }_cores`, buildImageTime ); increment( 'build.success' ); try { await refreshLocalImages(); diff --git a/src/config.ts b/src/config.ts index f6baa77..57c1e55 100644 --- a/src/config.ts +++ b/src/config.ts @@ -19,7 +19,7 @@ type RepoConfig = Readonly< { project: string; } >; -type EnvsConfig = Readonly; +type EnvsConfig = Readonly< RunEnv[] >; export const config: AppConfig = { build: { @@ -46,6 +46,6 @@ export function envContainerConfig( environment: RunEnv ): Dockerode.ContainerCr case 'jetpack': return { Env: [ 'NODE_ENV=jetpack-cloud-horizon', 'CALYPSO_ENV=jetpack-cloud-horizon' ], - } + }; } -} \ No newline at end of file +} diff --git a/src/middlewares.ts b/src/middlewares.ts index d54a3ba..9f0e7a9 100644 --- a/src/middlewares.ts +++ b/src/middlewares.ts @@ -3,17 +3,31 @@ import * as express from 'express'; import * as expressSession from 'express-session'; // internal -import { getCommitHashForBranch, refreshRemoteBranches, CommitHash, touchCommit, RunEnv } from './api'; +import { + getCommitHashForBranch, + refreshRemoteBranches, + CommitHash, + touchCommit, + RunEnv, +} from './api'; import { config } from './config'; const hashPattern = /(?:^|.*?\.)(\w*)-?hash-([a-f0-9]+)\./; -function assembleSubdomainUrlForHash( req: express.Request, commitHash: CommitHash, environment: RunEnv ) { +function assembleSubdomainUrlForHash( + req: express.Request, + commitHash: CommitHash, + environment: RunEnv +) { const protocol = req.secure || req.headers.host.indexOf( 'calypso.live' ) > -1 ? 'https' : 'http'; - const subdomainEnv = environment && environment !== config.envs[0] ? environment + '-' : ''; + const subdomainEnv = environment && environment !== config.envs[ 0 ] ? environment + '-' : ''; - const newUrl = new URL( `${protocol}://${subdomainEnv}hash-${commitHash}.${stripCommitHashSubdomainFromHost( req.headers.host )}` ); + const newUrl = new URL( + `${ protocol }://${ subdomainEnv }hash-${ commitHash }.${ stripCommitHashSubdomainFromHost( + req.headers.host + ) }` + ); newUrl.pathname = req.path; for ( let [ key, value ] of Object.entries( req.query ) ) { if ( key === 'hash' || key === 'branch' || key === 'env' ) { @@ -36,6 +50,8 @@ function getCommitHashFromSubdomain( host: string ) { return null; } + // https://github.com/prettier/prettier/issues/1013 + // prettier-ignore const [ /* full match */, /* environment */, hash ] = match; return hash; } @@ -47,6 +63,8 @@ function getEnvironmentFromSubdomain( host: string ) { return null; } + // https://github.com/prettier/prettier/issues/1013 + // prettier-ignore const [ /* full match */, environment ] = match; return environment; } @@ -119,13 +137,13 @@ export function determineCommitHash( export function determineEnvironment( req: express.Request, res: express.Response, - next: express.NextFunction, + next: express.NextFunction ) { const subdomainEnvironment = getEnvironmentFromSubdomain( req.headers.host ); if ( config.envs.includes( subdomainEnvironment ) ) { req.session.runEnv = subdomainEnvironment; } else { - req.session.runEnv = config.envs[0]; + req.session.runEnv = config.envs[ 0 ]; } next(); } From 435e0dd6688caf71d289dbc8d3e222fff8f50d93 Mon Sep 17 00:00:00 2001 From: Sergio Cinos Date: Tue, 10 Nov 2020 15:05:50 +0100 Subject: [PATCH 4/4] Support for running arbitrary images --- package.json | 1 + src/api.ts | 271 +++++++++++++++++++++++++++++++++++--------- src/app/debug.tsx | 26 ++++- src/config.ts | 12 +- src/image-runner.ts | 160 ++++++++++++++++++++++++++ src/index.ts | 11 +- src/middlewares.ts | 10 +- src/types.d.ts | 10 ++ test/api.test.ts | 87 +++++--------- yarn.lock | 5 + 10 files changed, 457 insertions(+), 136 deletions(-) create mode 100644 src/image-runner.ts create mode 100644 src/types.d.ts diff --git a/package.json b/package.json index ba63b01..22172e8 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@types/tar-fs": "^1.16.1", "@types/useragent": "^2.1.1", "bunyan": "^1.8.12", + "docker-parse-image": "^3.0.1", "dockerode": "^3.0.0", "express": "^4.16.3", "express-session": "^1.15.6", diff --git a/src/api.ts b/src/api.ts index e2aae98..45c1a36 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,5 +1,5 @@ import httpProxy from 'http-proxy'; -import Docker from 'dockerode'; +import Docker, { ImageInfo } from 'dockerode'; import _ from 'lodash'; import portfinder from 'portfinder'; import git from 'nodegit'; @@ -13,14 +13,15 @@ import { l } from './logger'; import { pendingHashes } from './builder'; import { exec } from 'child_process'; -import { CONTAINER_EXPIRY_TIME, START_TIME, TEN_MINUTES } from './constants'; +import { CONTAINER_EXPIRY_TIME } from './constants'; import { timing } from './stats'; type APIState = { - accesses: Map< CommitHash, number >; + accesses: Map< ContainerName, number >; branchHashes: Map< CommitHash, BranchName >; containers: Map< string, Docker.ContainerInfo >; - localImages: Map< string, Docker.ImageInfo >; + localImages: Map< ImageName, Docker.ImageInfo >; + pullingImages: Map< ImageName, Promise< DockerodeStream > >; remoteBranches: Map< BranchName, CommitHash >; startingContainers: Map< CommitHash, Promise< ContainerInfo > >; }; @@ -30,6 +31,7 @@ export const state: APIState = { branchHashes: new Map(), containers: new Map(), localImages: new Map(), + pullingImages: new Map(), remoteBranches: new Map(), startingContainers: new Map(), }; @@ -43,6 +45,17 @@ export type BranchName = string; export type PortNumber = number; export type ImageStatus = 'NoImage' | 'Inactive' | PortNumber; export type RunEnv = string; +export type DockerRepository = string; +export type ImageName = string; +export type ContainerName = string; +export type DockerodeStream = any; +export type ContainerSearchOptions = { + image?: ImageName; + env?: RunEnv; + status?: string; + id?: string; + name?: string; +}; export const getImageName = ( hash: CommitHash ) => `${ config.build.tagPrefix }:${ hash }`; export const extractCommitFromImage = ( imageName: string ): CommitHash => { @@ -58,30 +71,48 @@ export const extractEnvironmentFromImage = ( image: ContainerInfo ): RunEnv => { }; /** - * Polls the local Docker daemon to - * fetch an updated list of images + * Polls the local Docker daemon to fetch an updated list of images + * + * It saves them in the Map `stcate.localImages`, indexed by tag name. If an image has more than + * one tag it will appear multiple times in the map. */ export async function refreshLocalImages() { const images = await docker.listImages(); - const isTag = ( tag: string ) => tag.startsWith( config.build.tagPrefix ); - const hasTag = ( image: Docker.ImageInfo ) => image.RepoTags && image.RepoTags.some( isTag ); - state.localImages = new Map( - images - .filter( hasTag ) - .map( image => [ image.RepoTags.find( isTag ), image ] as [ string, Docker.ImageInfo ] ) + images.reduce( + ( acc, image ) => [ + ...acc, + ...( image.RepoTags || [] ).map( tag => [ tag, image ] as [ ImageName, ImageInfo ] ), + ], + [] + ) ); } /** - * Returns the list of local images + * Returns the list of images built by dserve */ export function getLocalImages() { + return new Map( + Array.from( state.localImages.entries() ).filter( ( [ imageName ] ) => + imageName.startsWith( config.build.tagPrefix ) + ) + ); +} + +/** + * Returns the list of all images + */ +export function getAllImages() { return state.localImages; } +export function getAllContainers() { + return state.containers; +} + export async function hasHashLocally( hash: CommitHash ): Promise< boolean > { - return state.localImages.has( getImageName( hash ) ); + return getLocalImages().has( getImageName( hash ) ); } export async function deleteImage( hash: CommitHash ) { @@ -196,7 +227,7 @@ export async function startContainer( commitHash: CommitHash, env: RunEnv ) { { image, freePort, commitHash }, `Successfully started container for ${ image } on ${ freePort }` ); - return refreshRunningContainers().then( () => getRunningContainerForHash( commitHash ) ); + return refreshContainers().then( () => getRunningContainerForHash( commitHash ) ); }, ( { error, freePort } ) => { l.error( @@ -224,8 +255,8 @@ export async function startContainer( commitHash: CommitHash, env: RunEnv ) { return startPromise; } -export async function refreshRunningContainers() { - const containers = await docker.listContainers(); +export async function refreshContainers() { + const containers = await docker.listContainers( { all: true } ); state.containers = new Map( containers.map( container => [ container.Id, container ] as [ string, ContainerInfo ] ) ); @@ -241,13 +272,6 @@ export function getRunningContainerForHash( hash: CommitHash, env?: RunEnv ): Co ); } -export function getRunningContainersForHash( hash: CommitHash ): ContainerInfo[] { - const image = getImageName( hash ); - return Array.from( state.containers.values() ).filter( - ci => ci.Image === image && ci.State === 'running' - ); -} - export function isContainerRunning( hash: CommitHash, env?: RunEnv ): boolean { return !! getRunningContainerForHash( hash, env ); } @@ -376,39 +400,29 @@ export function getCommitHashForBranch( branch: BranchName ): CommitHash | undef return state.remoteBranches.get( branch ); } -export function touchCommit( hash: CommitHash ) { - state.accesses.set( hash, Date.now() ); +export function touchCommit( hash: CommitHash, env?: RunEnv ) { + const container = getRunningContainerForHash( hash, env ); + if ( ! container ) throw `Running container for commit ${ hash } not found}`; + + const name = getContainerName( container ); + touchContainer( name ); } -export function getCommitAccessTime( hash: CommitHash ): number | undefined { - if ( ! hash ) { - return undefined; - } - return state.accesses.get( hash ); +export function touchContainer( name: ContainerName ) { + state.accesses.set( name, Date.now() ); +} + +export function getContainerAccessTime( name: ContainerName ): number | undefined { + return state.accesses.get( name ); } /* - * Get all currently running containers that were created by dserve and have expired. + * Get all currently running containers that have expired. * Expired means have not been accessed in EXPIRED_DURATION */ -export function getExpiredContainers( - containers: Array< ContainerInfo >, - getAccessTime: Function -) { - // if the server is newly spun up, wait a bit before killing off running containers - if ( Date.now() - START_TIME < TEN_MINUTES ) { - return []; - } - - // otherwise, filter off containers that are still valid - return containers.filter( ( container: ContainerInfo ) => { - const imageName: string = container.Image; - - // exclude container if it wasnt created by this app - // if ( ! imageName.startsWith( config.build.tagPrefix ) ) { - // return false; - // } - +export function getExpiredContainers() { + // Filter off containers that are still valid + return Array.from( state.containers.values() ).filter( ( container: ContainerInfo ) => { if ( container.State === 'dead' || container.State === 'created' ) { // ignore dead and just created containers return false; @@ -420,7 +434,7 @@ export function getExpiredContainers( } const createdAgo = Date.now() - container.Created * 1000; - const lastAccessed = getAccessTime( extractCommitFromImage( imageName ) ); + const lastAccessed = getContainerAccessTime( getContainerName( container ) ); return ( createdAgo > CONTAINER_EXPIRY_TIME && @@ -431,8 +445,8 @@ export function getExpiredContainers( // stop any container that hasn't been accessed within ten minutes export async function cleanupExpiredContainers() { - const containers = Array.from( await docker.listContainers( { all: true } ) ); - const expiredContainers = getExpiredContainers( containers, getCommitAccessTime ); + await refreshContainers(); + const expiredContainers = getExpiredContainers(); for ( let container of expiredContainers ) { const imageName: string = container.Image; @@ -459,7 +473,7 @@ export async function cleanupExpiredContainers() { l.error( { err, imageName, containerId: container.Id }, 'Failed to remove container' ); } } - refreshRunningContainers(); + refreshContainers(); } const proxy = httpProxy.createProxyServer( {} ); // See (†) @@ -474,6 +488,7 @@ export async function proxyRequestToHash( req: any, res: any ) { return; } + touchCommit( commitHash, runEnv ); proxy.web( req, res, { target: `http://localhost:${ port }` }, err => { if ( err && ( err as any ).code === 'ECONNRESET' ) { return; @@ -481,3 +496,151 @@ export async function proxyRequestToHash( req: any, res: any ) { l.log( { err, req, res, commitHash }, 'unexpected error occured while proxying' ); } ); } + +export function getContainerName( container: ContainerInfo ) { + // The first character is a `/`, skip it + return container.Names[ 0 ].substring( 1 ); +} + +export function findContainer( { id, image, env, status, name }: ContainerSearchOptions ) { + return Array.from( state.containers.values() ).find( container => { + if ( image && ( container.Image !== image && container.ImageID !== image ) ) return false; + if ( env && container.Labels[ 'calypsoEnvironment' ] !== env ) return false; + if ( status && container.Status !== status ) return false; + if ( id && container.Id !== id ) return false; + // In the Docker internal list, names start with `/` + if ( name && ! container.Names.includes( '/' + name ) ) return false; + return true; + } ); +} + +export async function proxyRequestToContainer( req: any, res: any, container: ContainerInfo ) { + // In the Docker internal list, names start with `/` + const containerName = getContainerName( container ); + + if ( ! container.Ports[ 0 ] ) { + l.log( { containerName }, `Could not find port for container` ); + throw new Error( `Could not find port for container ${ containerName }` ); + } + const port = container.Ports[ 0 ].PublicPort; + + let retryCounter = config.proxyRetry; + const proxyToContainer = () => + proxy.web( req, res, { target: `http://localhost:${ port }` }, errorHandler ); + const errorHandler = ( err: any ) => { + if ( err && ( err as any ).code === 'ECONNRESET' ) { + retryCounter--; + if ( retryCounter > 0 ) setTimeout( proxyToContainer, 1000 ); + } + l.log( { err, req, res, containerName }, 'unexpected error occured while proxying' ); + throw new Error( 'unexpected error occured while proxying' ); + }; + touchContainer( containerName ); + proxyToContainer(); +} + +/** + * Pulls an image. Calls onProgress() when there is an update, resolves the returned promise + * when the image is pulled + */ +export async function pullImage( imageName: ImageName, onProgress: ( data: any ) => void ) { + // Store the stream in memory, so other requets can "join" and listen for the progress + if ( ! state.pullingImages.has( imageName ) ) { + const stream = docker.pull( imageName, {} ) as Promise< DockerodeStream >; + state.pullingImages.set( imageName, stream ); + } + + const stream = state.pullingImages.get( imageName ); + return new Promise( async ( resolve, reject ) => { + const resolvedStream = await stream; + + docker.modem.followProgress( + resolvedStream, + ( err: any ) => { + state.pullingImages.delete( imageName ); + if ( err ) reject( err ); + else resolve(); + }, + onProgress + ); + } ); +} + +/** + * Asks a container nicely to stop, waits for 10 seconds and then obliterates it + */ +export async function deleteContainer( containerInfo: ContainerInfo ) { + const container = docker.getContainer( containerInfo.Id ); + if ( containerInfo.State === 'running' ) { + await container.stop( { t: 10 } ); + } + await container.remove( { force: true } ); + await refreshContainers(); +} + +/** + * Creates a container + * + * createContainer is async, but we don't keep a list of container being creates to ensure atomicity for a few reasons: + * + * - Creating container is quite fast (a few ms), so the chances of collisions are quite low + * - Even if we get two requests with the same image+env at the same time, creating two separate containers for the same + * image is ok. Each one will get a different URL, and if one of them is not used it will get eventually cleaned up. + */ +export async function createContainer( imageName: ImageName, env: RunEnv ) { + const exposedPort = `${ config.build.exposedPort }/tcp`; + + let freePort: number; + try { + freePort = await portfinder.getPortPromise(); + } catch ( err ) { + l.error( { err, imageName }, `Error while attempting to find a free port for ${ imageName }` ); + throw err; + } + + try { + const container = await docker.createContainer( { + ...config.build.containerCreateOptions, + ...envContainerConfig( env ), + Image: imageName, + ExposedPorts: { [ exposedPort ]: {} }, + HostConfig: { + PortBindings: { [ exposedPort ]: [ { HostPort: freePort.toString() } ] }, + }, + Labels: { + calypsoEnvironment: env, + }, + } ); + l.log( { imageName }, `Successfully created container for ${ imageName }` ); + await refreshContainers(); + + // Returns a ContainerInfo for the created container, in order to avoid exposing a real Container object. + return findContainer( { + id: container.id, + } ); + } catch ( error ) { + l.error( { imageName, error }, `Failed creating container for ${ imageName }` ); + throw error; + } +} + +/** + * Starts a container that was dormant (either never started, or stopped) + */ +export async function reviveContainer( containerInfo: ContainerInfo ) { + const containerName = getContainerName( containerInfo ); + const container = docker.getContainer( containerInfo.Id ); + + try { + await container.start(); + await refreshContainers(); + + // This returns the same containerInfo object, but updated + return findContainer( { + id: container.id, + } ); + } catch ( error ) { + l.error( { containerName, error }, `Failed starting container ${ containerName }` ); + throw error; + } +} diff --git a/src/app/debug.tsx b/src/app/debug.tsx index 48ed9c9..8eeb3e5 100644 --- a/src/app/debug.tsx +++ b/src/app/debug.tsx @@ -8,7 +8,14 @@ import { Shell } from './app-shell'; import { promiseRejections } from '../index'; import { humanSize, humanRelativeTime, percent, round } from './util'; -import { state as apiState, getCommitAccessTime, extractCommitFromImage, extractEnvironmentFromImage } from '../api'; +import { + getAllContainers, + getLocalImages, + extractCommitFromImage, + extractEnvironmentFromImage, + getContainerAccessTime, + getContainerName, +} from '../api'; import { buildQueue, pendingHashes } from '../builder'; const Docker = new Dockerode(); @@ -26,10 +33,10 @@ const Debug = ( c: RenderContext ) => { const heapT = memUsage.heapTotal; const memTotal = os.totalmem(); const memUsed = memTotal - os.freemem(); - const images = Array.from( apiState.localImages.entries() ) as Array< + const images = Array.from( getLocalImages().entries() ) as Array< [ string, Dockerode.ImageInfo ] >; - const apiContainers = Array.from( apiState.containers.entries() ); + const apiContainers = Array.from( getAllContainers().entries() ); const shortHash = ( hash: string = '', length = 30 ) => ( { hash.slice( 0, length ) }… @@ -208,7 +215,12 @@ const Debug = ( c: RenderContext ) => {
  • { info.Names } - { shortHash( info.Id ) }
    - Commit: { commit ? { commit } : 'none' } + Commit:{' '} + { commit ? ( + { commit } + ) : ( + 'none' + ) }
    Image ID: { shortHash( info.ImageID ) }
    @@ -217,8 +229,10 @@ const Debug = ( c: RenderContext ) => { Status: { info.Status }
    Last Access:{' '} - { getCommitAccessTime( commit ) - ? humanRelativeTime( getCommitAccessTime( commit ) / 1000 ) + { getContainerAccessTime( getContainerName( info ) ) + ? humanRelativeTime( + getContainerAccessTime( getContainerName( info ) ) / 1000 + ) : 'never' }
  • ); diff --git a/src/config.ts b/src/config.ts index b1c378c..5b65f93 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,11 +1,13 @@ import Dockerode from 'dockerode'; -import { RunEnv } from './api'; +import { DockerRepository, RunEnv } from './api'; type Readonly< T > = { readonly [ P in keyof T ]: T[ P ] }; type AppConfig = Readonly< { build: BuildConfig; repo: RepoConfig; envs: EnvsConfig; + allowedDockerRepositories: AllowedDockerRepositories; + proxyRetry: number; } >; type BuildConfig = Readonly< { @@ -21,6 +23,8 @@ type RepoConfig = Readonly< { type EnvsConfig = Readonly< RunEnv[] >; +type AllowedDockerRepositories = Readonly< DockerRepository[] >; + export const config: AppConfig = { build: { containerCreateOptions: {}, @@ -34,6 +38,12 @@ export const config: AppConfig = { }, envs: [ 'calypso', 'jetpack' ], + + allowedDockerRepositories: [ 'registry.a8c.com' ], + + // When the proxy to the container fails with a ECONNRESET error, retry this number + // of times. + proxyRetry: 3, }; export function envContainerConfig( environment: RunEnv ): Dockerode.ContainerCreateOptions { diff --git a/src/image-runner.ts b/src/image-runner.ts new file mode 100644 index 0000000..91cac63 --- /dev/null +++ b/src/image-runner.ts @@ -0,0 +1,160 @@ +// external +import express from 'express'; +import { ContainerInfo } from 'dockerode'; + +// internal +import { + getAllImages, + findContainer, + pullImage, + deleteContainer, + ContainerName, + proxyRequestToContainer, + reviveContainer, + createContainer, + getContainerName, +} from './api'; +import { config } from './config'; +import { l } from './logger'; +import dockerParseImage from 'docker-parse-image'; + +const imagePattern = /^container-(?\w+)\./; + +function stripImageHashSubdomainFromHost( host: string ) { + return host.replace( imagePattern, '' ); +} + +function assembleSubdomainUrlForContainer( req: express.Request, container: ContainerInfo ) { + const protocol = req.secure || req.headers.host.indexOf( 'calypso.live' ) > -1 ? 'https' : 'http'; + const environment = container.Labels[ 'calypsoEnvironment' ]; + + const subdomainEnv = environment && environment !== config.envs[ 0 ] ? environment + '-' : ''; + const name = getContainerName( container ); + + const newUrl = new URL( + `${ protocol }://${ subdomainEnv }container-${ name }.${ stripImageHashSubdomainFromHost( + req.headers.host + ) }` + ); + newUrl.pathname = req.path; + for ( let [ key, value ] of Object.entries( req.query ) ) { + if ( key === 'hash' || key === 'branch' || key === 'env' || key === 'image' ) { + continue; + } + newUrl.searchParams.set( key, String( value ) ); + } + + return newUrl.toString(); +} + +function getContainerNameFromSubdomain( host: string ) { + const match = host.match( imagePattern ); + if ( ! match ) { + return null; + } + + return match.groups.container; +} + +/** + * Gets an image name from the query string, finds (or creates) a container for that image + * and redirects to http://container-.calypso.live + */ +async function loadImage( req: express.Request, res: express.Response ) { + res.header( 'Cache-control', 'no-cache' ); + + const imageName = req.query.image; + const environment = req.query.env || config.envs[ 0 ]; + + const { registry } = dockerParseImage( imageName ); + if ( ! config.allowedDockerRepositories.includes( registry ) ) { + res.status( 403 ).send( `The registry ${ registry } is invalid` ); + return; + } + + // There is a container for this image/environment. Redirect to http://container-.calypso.live + const existingContainer = findContainer( { + image: imageName, + env: environment, + } ); + if ( existingContainer ) { + res.redirect( assembleSubdomainUrlForContainer( req, existingContainer ) ); + return; + } + + // There is no a container for this image, but the image exists in our repo. Create the container and redirect + if ( getAllImages().has( imageName ) ) { + const container = await createContainer( imageName, environment ); + res.redirect( assembleSubdomainUrlForContainer( req, container ) ); + return; + } + + // Neither the container nor the image exits. Pull the image, create the container and redirect. If they image is + // already being pulled, this will "attach" to the output of the existing pull. + res.status( 202 ); + res.write( '
    ' );
    +	await pullImage( imageName, data => {
    +		res.write( `${ Date.now() } - ${ JSON.stringify( data ) }\n` );
    +	} );
    +	const container = await createContainer( imageName, environment );
    +	const url = assembleSubdomainUrlForContainer( req, container );
    +	res.write(
    +		`
    ` + ); + res.end(); +} + +/** + * Gets a container name from the subdmain, starts it if necessary and proxies all requests to it. + */ +async function startAndProxyRequestsToContainer( req: express.Request, res: express.Response ) { + const containerName: ContainerName = getContainerNameFromSubdomain( req.headers.host ); + let container: ContainerInfo; + const shouldDelete = 'delete' in req.query; + + container = findContainer( { + name: containerName, + } ); + if ( ! container ) { + throw new Error( `Container ${ containerName } not found` ); + } + + if ( shouldDelete ) { + l.log( { containerName }, `Hard reset for ${ containerName }` ); + await deleteContainer( container ); + res.send( `Container ${ containerName } deleted` ); + return; + } + + if ( container.State !== 'running' ) { + container = await reviveContainer( container ); + } + + proxyRequestToContainer( req, res, container ); +} + +/** + * Main middleware for the image runner. This single middleware will take care of both cases: + * + * - Image in query string (http://calypso.live?/image=calypso/app:build-4) + * - Container name in subdomain (http://container-agitated_hypatia.calypso.live/) + */ +export function middleware( + req: express.Request, + res: express.Response, + next: express.NextFunction +) { + const imageName = req.query && req.query.image; + if ( imageName ) { + loadImage( req, res ).catch( next ); + return; + } + + const containerName = getContainerNameFromSubdomain( req.headers.host ); + if ( containerName ) { + startAndProxyRequestsToContainer( req, res ).catch( next ); + return; + } + + next(); +} diff --git a/src/index.ts b/src/index.ts index 2000a33..7796bf2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,7 +21,7 @@ import { getBranchHashes, } from './api'; -import { ONE_MINUTE, ONE_SECOND } from './constants'; +import { ONE_MINUTE, ONE_SECOND, TEN_MINUTES } from './constants'; import { isBuildInProgress, @@ -39,7 +39,7 @@ import { session, determineEnvironment, } from './middlewares'; - +import { middleware as imageRunnerMiddleware } from './image-runner'; import renderApp from './app/index'; import renderLocalImages from './app/local-images'; import renderLog from './app/log'; @@ -51,7 +51,7 @@ import { Writable } from 'stream'; import { refreshLocalImages, refreshRemoteBranches, - refreshRunningContainers, + refreshContainers, cleanupExpiredContainers, } from './api'; @@ -142,6 +142,7 @@ calypsoServer.get( '/debug', async ( req: express.Request, res: express.Response } } ); +calypsoServer.use( imageRunnerMiddleware ); calypsoServer.use( redirectHashFromQueryStringToSubdomain ); calypsoServer.use( determineCommitHash ); calypsoServer.use( determineEnvironment ); @@ -270,9 +271,9 @@ if ( process.env.NODE_ENV !== 'test' ) { }; loop( refreshLocalImages, 5 * ONE_SECOND ); - loop( refreshRunningContainers, 5 * ONE_SECOND ); + loop( refreshContainers, 5 * ONE_SECOND ); loop( refreshRemoteBranches, ONE_MINUTE ); // Wait a bit before starting the expired container cleanup. // This gives us some time to accumulate accesses to existing containers across app restarts - setTimeout( () => loop( cleanupExpiredContainers, ONE_MINUTE ), 2 * ONE_MINUTE ); + setTimeout( () => loop( cleanupExpiredContainers, ONE_MINUTE ), TEN_MINUTES ); } diff --git a/src/middlewares.ts b/src/middlewares.ts index efcf33f..4e7933b 100644 --- a/src/middlewares.ts +++ b/src/middlewares.ts @@ -3,13 +3,7 @@ import express from 'express'; import expressSession from 'express-session'; // internal -import { - getCommitHashForBranch, - refreshRemoteBranches, - CommitHash, - touchCommit, - RunEnv, -} from './api'; +import { getCommitHashForBranch, refreshRemoteBranches, CommitHash, RunEnv } from './api'; import { config } from './config'; const hashPattern = /(?:^|.*?\.)(\w*)-?hash-([a-f0-9]+)\./; @@ -129,8 +123,6 @@ export function determineCommitHash( req.session.commitHash = subdomainCommitHash; - touchCommit( subdomainCommitHash ); - next(); } diff --git a/src/types.d.ts b/src/types.d.ts new file mode 100644 index 0000000..0615573 --- /dev/null +++ b/src/types.d.ts @@ -0,0 +1,10 @@ +declare module 'docker-parse-image' { + export default function parse( + imgName: string + ): { + registry: string; + namespace: string; + repository: string; + tag: string; + }; +} diff --git a/test/api.test.ts b/test/api.test.ts index 1e384e2..f98d924 100644 --- a/test/api.test.ts +++ b/test/api.test.ts @@ -1,4 +1,4 @@ -import { getCommitAccessTime, touchCommit, getExpiredContainers, getImageName } from '../src/api'; +import { getExpiredContainers, getImageName, state } from '../src/api'; import { CONTAINER_EXPIRY_TIME } from '../src/constants'; @@ -7,88 +7,53 @@ describe( 'api', () => { const RealNow = Date.now; const fakeNow = RealNow() + 24 * 60 * 1000; Date.now = () => fakeNow; + afterAll( () => { Date.now = RealNow; } ); const EXPIRED_TIME = Date.now() - CONTAINER_EXPIRY_TIME - 1; const GOOD_TIME = Date.now() - CONTAINER_EXPIRY_TIME + 1; - const images = [ { Image: getImageName( '1' ), Id: 1 }, { Image: getImageName( '2' ), Id: 2 } ]; + const images = [ + { Image: getImageName( '1' ), Id: 1, Created: EXPIRED_TIME / 1000, Names: [ '/foo' ] }, + { Image: getImageName( '2' ), Id: 1, Created: EXPIRED_TIME / 1000, Names: [ '/bar' ] }, + ]; - test( 'returns nothing for empty list of containers', () => { - expect( getExpiredContainers( [], () => 0 ) ).toEqual( [] ); + afterEach( () => { + state.accesses = new Map(); + state.containers = new Map(); } ); - test( 'returns the whole list if everything is expired', () => { - const expiredImages = images.map( image => { - return Object.assign( {}, image, { Created: EXPIRED_TIME / 1000 } ); - } ); - expect( getExpiredContainers( expiredImages as any, () => CONTAINER_EXPIRY_TIME ) ).toEqual( - expiredImages - ); + beforeEach( () => { + state.containers = new Map( images.map( image => [ image.Image, { ...image } ] ) as any ); } ); - test( 'returns empty list if everything is before expiry', () => { - expect( getExpiredContainers( images as any, () => GOOD_TIME ) ).toEqual( [] ); + test( 'returns nothing for empty list of containers', () => { + state.containers = new Map(); + expect( getExpiredContainers() ).toEqual( [] ); } ); - test( 'returns list of only images that have not expired', () => { - const getAccessTime = jest - .fn() - .mockReturnValueOnce( EXPIRED_TIME ) - .mockReturnValueOnce( GOOD_TIME ); - const oldImages = images.map( image => { - return Object.assign( {}, image, { Created: EXPIRED_TIME / 1000 } ); - } ); - expect( getExpiredContainers( oldImages as any, getAccessTime ) ).toEqual( - [].concat( oldImages[ 0 ] ) - ); + test( 'returns the whole list if everything is expired', () => { + expect( getExpiredContainers() ).toEqual( images ); } ); - test( 'young images are not returned, regardless of access time', () => { - const expiredImages = images.map( image => { - return Object.assign( {}, image, { Created: EXPIRED_TIME / 1000 } ); - } ); - expiredImages[ 0 ].Created = Date.now() / 1000; - expect( getExpiredContainers( expiredImages as any, () => CONTAINER_EXPIRY_TIME ) ).toEqual( - [].concat( expiredImages[ 1 ] ) - ); - } ); - } ); + test.only( 'returns empty list if everything was accessed before expiry', () => { + state.accesses.set( 'foo', GOOD_TIME ); + state.accesses.set( 'bar', GOOD_TIME ); - describe( 'commitAccessTimes', () => { - beforeEach( () => { - jest.resetModules(); - } ); - test( 'should return undefined for non-existent hash', () => { - expect( getCommitAccessTime( 'nanana' ) ).toBe( undefined ); + expect( getExpiredContainers() ).toEqual( [] ); } ); - test( 'should return a date for touched commit', () => { - touchCommit( 'touched' ); - expect( getCommitAccessTime( 'touched' ) ).toEqual( expect.any( Number ) ); - } ); + test( 'returns list of only images that have not expired', () => { + state.accesses.set( 'foo', Date.now() ); - test( 'should return same time for a commit that doesnt get touched again', () => { - touchCommit( 'touched' ); - const touch1 = getCommitAccessTime( 'touched' ); - touchCommit( 'nanan' ); - expect( getCommitAccessTime( 'touched' ) ).toBe( touch1 ); + expect( getExpiredContainers() ).toEqual( [ state.containers.get( getImageName( '2' ) ) ] ); } ); - test( 'should update a commit touch date to be newer if called again', () => { - const RealNow = Date.now; - - let count = 0; - Date.now = jest.fn().mockImplementation( () => count++ ); - - touchCommit( 'touched' ); - const touch1 = getCommitAccessTime( 'touched' ); - touchCommit( 'touched' ); - const touch2 = getCommitAccessTime( 'touched' ); - expect( touch2 ).toBeGreaterThan( touch1 ); + test( 'young images are not returned, regardless of access time', () => { + state.containers.get( getImageName( '1' ) ).Created = Date.now() / 1000; - Date.now = RealNow; + expect( getExpiredContainers() ).toEqual( [ state.containers.get( getImageName( '2' ) ) ] ); } ); } ); } ); diff --git a/yarn.lock b/yarn.lock index 3fe8b66..e7feec9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1377,6 +1377,11 @@ docker-modem@^2.0.0: split-ca "^1.0.0" ssh2 "^0.8.5" +docker-parse-image@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/docker-parse-image/-/docker-parse-image-3.0.1.tgz#33dc69291eac3414f84871f2d59d77b6f6948be4" + integrity sha1-M9xpKR6sNBT4SHHy1Z13tvaUi+Q= + dockerode@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-3.0.2.tgz#6e61de42ecbbae997196874e53150e3d7ae3c964"