diff --git a/README.md b/README.md index 99eb6c26..a90ce26d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,11 @@ You can run `docker build` and `docker-compose build` in your GitHub Actions wor This GitHub Action uses the [docker save](https://docs.docker.com/engine/reference/commandline/save/) / [docker load](https://docs.docker.com/engine/reference/commandline/load/) command and the [@actions/cache](https://www.npmjs.com/package/@actions/cache) library. +## ⚠️ **Deprecation Notice for `v0.0.4` and older** ⚠️ + +The author had not taken into account that a large number of layers would be cached, +so those versions processes all layers in parallel. ([#12](https://github.com/satackey/action-docker-layer-caching/issues/12)) +**Please update to version `v0.0.5` with limited concurrency to avoid overloading the cache service.** ## Example workflows @@ -30,11 +35,11 @@ jobs: # In this step, this action saves a list of existing images, # the cache is created without them in the post run. # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.4 + - uses: satackey/action-docker-layer-caching@v0.0.5 - run: docker-compose up --build - # Finally, "Post Run satackey/action-docker-layer-caching@v0.0.4", + # Finally, "Post Run satackey/action-docker-layer-caching@v0.0.5", # which is the process of saving the cache, will be executed. ``` @@ -56,12 +61,12 @@ jobs: # In this step, this action saves a list of existing images, # the cache is created without them in the post run. # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.4 + - uses: satackey/action-docker-layer-caching@v0.0.5 - name: Build the Docker image run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) - # Finally, "Post Run satackey/action-docker-layer-caching@v0.0.4", + # Finally, "Post Run satackey/action-docker-layer-caching@v0.0.5", # which is the process of saving the cache, will be executed. ``` @@ -74,7 +79,7 @@ By default, the cache is separated by the workflow name. You can also set the cache key manually, like the official [actions/cache](https://github.com/actions/cache#usage) action. ```yaml - - uses: satackey/action-docker-layer-caching@v0.0.4 + - uses: satackey/action-docker-layer-caching@v0.0.5 with: key: foo-docker-cache-{hash} restore-keys: | diff --git a/action.yml b/action.yml index 97abde2a..e3dbbbfd 100644 --- a/action.yml +++ b/action.yml @@ -14,6 +14,10 @@ inputs: description: An ordered list of keys to use for restoring the cache if no cache hit occurred for key required: false default: docker-layer-caching-${{ github.workflow }}- + concurrency: + description: The number of concurrency when restoring and saving layers + required: true + default: '4' runs: using: node12 diff --git a/main.ts b/main.ts index 9e2ebbe3..65ad632a 100644 --- a/main.ts +++ b/main.ts @@ -11,6 +11,7 @@ const main = async () => { core.saveState(`already-existing-images`, JSON.stringify(await new ImageDetector().getExistingImages())) const layerCache = new LayerCache([]) + layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) const restoredKey = await layerCache.restore(primaryKey, restoreKeys) await layerCache.cleanUp() diff --git a/package.json b/package.json index f3790328..85879ea7 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@actions/core": "^1.2.4", "actions-exec-listener": "^0.0.2", "crypto": "^1.0.1", + "native-promise-pool": "^3.13.0", "string-format": "^2.0.0", "typescript-is": "^0.16.3" }, diff --git a/post.ts b/post.ts index 9d292f0e..09593358 100644 --- a/post.ts +++ b/post.ts @@ -18,6 +18,7 @@ const main = async () => { await imageDetector.getExistingImages() core.debug(JSON.stringify({ imageIdsToSave: imageDetector.getImagesShouldSave() })) const layerCache = new LayerCache(imageDetector.getImagesShouldSave()) + layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) layerCache.unformattedOrigianlKey = primaryKey core.debug(JSON.stringify({ restoredKey, formattedOriginalCacheKey: layerCache.getFormattedOriginalCacheKey()})) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 4939e98e..5251ef68 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -7,6 +7,7 @@ import { ExecOptions } from '@actions/exec/lib/interfaces' import { promises as fs } from 'fs' import { assertManifests, Manifest, Manifests } from './Tar' import format from 'string-format' +import PromisePool from 'native-promise-pool' class LayerCache { // repotag: string @@ -18,6 +19,7 @@ class LayerCache { enabledParallel = true // unpackedTarDir: string = '' // manifests: Manifests = [] + concurrency: number = 4 constructor(ids: string[]) { // this.repotag = repotag @@ -108,7 +110,16 @@ class LayerCache { } private async storeLayers(): Promise { - return await Promise.all((await this.getLayerIds()).map(layerId => this.storeSingleLayerBy(layerId))) + const pool = new PromisePool(this.concurrency) + + const result = Promise.all( + (await this.getLayerIds()).map( + layerId => { + return pool.open(() => this.storeSingleLayerBy(layerId)) + } + ) + ) + return result } static async dismissCacheAlreadyExistsError(promise: Promise): Promise { @@ -173,16 +184,31 @@ class LayerCache { } private async restoreLayers() { - const restoring = (await this.getLayerIds()).map(layerId => this.restoreSingleLayerBy(layerId)); - const restoredLayerKeysThatMayContainUndefined = await Promise.all(restoring) + const pool = new PromisePool(this.concurrency) + + const restoredLayerKeysThatMayContainUndefined = await Promise.all( + (await this.getLayerIds()).map( + layerId => { + return pool.open(() => this.restoreSingleLayerBy(layerId)) + } + ) + ) + core.debug(JSON.stringify({ log: `restoreLayers`, restoredLayerKeysThatMayContainUndefined })) const FailedToRestore = (restored: string | undefined) => restored === undefined return restoredLayerKeysThatMayContainUndefined.filter(FailedToRestore).length === 0 } - private async restoreSingleLayerBy(id: string): Promise { + private async restoreSingleLayerBy(id: string): Promise { core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id })) - return await cache.restoreCache([this.genSingleLayerStorePath(id)], this.genSingleLayerStoreKey(id)) + + const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], this.genSingleLayerStoreKey(id)) + + if (result == null) { + throw new Error(`Layer cache not found: ${JSON.stringify({ id })}`) + } + + return result } private async loadImageFromUnpacked() { diff --git a/yarn.lock b/yarn.lock index 584eab5a..4f1a3635 100644 --- a/yarn.lock +++ b/yarn.lock @@ -329,6 +329,11 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +native-promise-pool@^3.13.0: + version "3.13.0" + resolved "https://registry.yarnpkg.com/native-promise-pool/-/native-promise-pool-3.13.0.tgz#faeaf9b4ee769a79328cc0435930079ad951b27b" + integrity sha512-CnUhv57AXMyxGazhYrqhlhwRg1rG36ygrDlnaCa70jBOePhV+OLM9CZkUAgceTXO6BZEQ3F5GRRWwB+wSzI4Aw== + nested-error-stacks@^2: version "2.1.0" resolved "https://registry.yarnpkg.com/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz#0fbdcf3e13fe4994781280524f8b96b0cdff9c61"