From 18af5a1f48680b6438d09e960e8e857640b1a330 Mon Sep 17 00:00:00 2001 From: mbshields Date: Tue, 24 Oct 2023 12:59:58 -0700 Subject: [PATCH 1/4] docs: add article for Retention Policies Signed-off-by: mbshields --- docs/articles/retention.md | 183 +++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 184 insertions(+) create mode 100644 docs/articles/retention.md diff --git a/docs/articles/retention.md b/docs/articles/retention.md new file mode 100644 index 0000000..64794b2 --- /dev/null +++ b/docs/articles/retention.md @@ -0,0 +1,183 @@ +# Configuring zot Tag Retention Policies + +> :point_right: To optimize image storage, you can configure tag retention policies to remove images that are no longer needed. + +Tag retention policies in zot can specify how many tags of a given repository to retain or how long to retain certain tags. + +You can define tag retention policies that apply one or more of the following rules: + +- Top tags most recently pushed +- Top tags most recently pulled +- Tags pushed in the past hours +- Tags pulled in the past hours +- Tags matching a regular expression (regex) pattern + + +## Configuring retention policies + +Retention policies are configured in the `storage` section of the zot configuration file under the `retention` attribute. One or more policies can be grouped under the `policies` attribute. + +By default, if no retention policies are defined, all tags are retained. + +> :warning: If at least one retention policy is defined, all tags not matching any policy are removed. We recommend defining a default policy to avoid unintended removals. + +### Configuration example + +The following example is a simple retention configuration with two policies: + +- The first includes all available configuration attributes. +- The second acts as a default policy. + +_simple policy example_ + +```json + "storage": { + "retention": { + "dryRun": false, + "delay": "24h", + "policies": [ + { + "repoNames": ["infra/*", "tmp/**"], + "deleteReferrers": false, + "deleteUntagged": true, + "KeepTags": [{ + "patterns": ["v2.*", ".*-prod"], + "mostRecentlyPushedCount": 10, + "mostRecentlyPulledCount": 10, + "pulledWithin": "720h", + "pushedWithin": "720h" + }] + }, + { + "keepTags": [{ + "patterns": [".*"] + }] + } + ] + } + } +``` + +### Configurable attributes + +The following table lists the attributes available in the retention policy configuration. + +| Attribute | Value | Description | +|-----------| ----- | ------------| +| dryRun | boolean | If `true`, will log a removal action without actually removing the image. Default is `false`. | +| delay | time | Remove untagged and referrers only if they are older than the specified hours, such as 24h. | +| policies | list | A list of policies. | +| repositories | list | A list of glob patterns to match repositories. | +| deleteReferrers | boolean | If true, delete manifests with a missing Subject. Default is `false`. | +| deleteUntagged | boolean | If true, delete untagged manifests. Default is `true`. | +| keepTags | list | Criteria for tags to retain always. | +| mostRecentlyPushedCount | count | Retains the top most recently pushed tags. | +| mostRecentlyPulledCount | count | Retains the top most recently pulled tags. | +| pushedWithin | time | Retains the tags pushed during the last hours, such as 24h. | +| pulledWithin | time | Retains the tags pulled during the last hours, such as 24h. | +| patterns | regex | See Notes. | + +> Mike's questions: +> - Which is the correct attribute name: `repositories` (as in config-retention.json) or `repoNames` (as in README.md)? +> - Are these statements conflicting? +> +> - If at least one retention policy is defined, all tags not matching any policy are removed. +> - If a repository matches no policy, the repository and all its tags are retained. + +### Configuration notes + +- A repository will apply the first policy it matches. +- If a repository matches no policy, the repository and all its tags are retained. +- If `keepTags` is present but empty, all tags are retained. +- When multiple rules are configured, a tag is retained if it meets at least one rule. +- When you specify a regex pattern combined with one or more rules, the rules are applied only to those tags matching the regex. +- When you specify a regex pattern with no rules other than the default, all tags matching the pattern are retained. +- In the repositories list, a single asterisk (/\*) matches all first-level items in the repository. A double asterisk (/*\*) matches all recursively. + +:warning: If at least one retention policy is defined, all tags not matching any policy are removed. We recommend defining a default policy, such as the following example, as the last policy in the policy list. All tags that don't match the preceding policies will be retained by this default policy: + +_default policy example_ + +```json + { + "keepTags": [{ + "patterns": [".*"] + }] + } +``` + + +## Complete configuration file example + +The following example shows the configuration of multiple retention policies in the context of a complete configuration file. + +```json +{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "/tmp/zot", + "gc": true, + "gcDelay": "2h", + "gcInterval": "1h", + "retention": { + "dryRun": false, + "delay": "24h", + "policies": [ + { + "repositories": ["infra/*", "prod/*"], + "deleteReferrers": false, + "keepTags": [{ + "patterns": ["v2.*", ".*-prod"] + }, + { + "patterns": ["v3.*", ".*-prod"], + "pulledWithin": "168h" + }] + }, + { + "repositories": ["tmp/**"], + "deleteReferrers": true, + "deleteUntagged": true, + "keepTags": [{ + "patterns": ["v1.*"], + "pulledWithin": "168h", + "pushedWithin": "168h" + }] + }, + { + "repositories": ["**"], + "deleteReferrers": true, + "deleteUntagged": true, + "keepTags": [{ + "mostRecentlyPushedCount": 10, + "mostRecentlyPulledCount": 10, + "pulledWithin": "720h", + "pushedWithin": "720h" + }] + } + ] + }, + "subPaths": { + "/a": { + "rootDirectory": "/tmp/zot1", + "dedupe": true, + "retention": { + "policies": [ + { + "repositories": ["infra/*", "prod/*"], + "deleteReferrers": false + } + ] + } + } + } + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + } +} +``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index acba44e..0e29551 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -118,6 +118,7 @@ nav: - Software Provenance Workflow Using OCI Artifacts: articles/workflow.md - Security Posture: articles/security-posture.md - Storage Planning: articles/storage.md + - Retention Policies: articles/retention.md - Mirroring: articles/mirroring.md - Clustering: articles/clustering.md - Monitoring: articles/monitoring.md From 6d1e29baf0d5ad83db06c45af44b619a923cfd0c Mon Sep 17 00:00:00 2001 From: mbshields Date: Tue, 24 Oct 2023 13:07:57 -0700 Subject: [PATCH 2/4] docs: add article for Retention Policies - spellcheck Signed-off-by: mbshields --- .wordlist.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.wordlist.txt b/.wordlist.txt index 8a30b43..71ef93b 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -16,6 +16,7 @@ bcrypt benchmarking blackbox BoltDB +boolean busybox CD certDir @@ -50,11 +51,14 @@ datapath DCO deduplicates deduplication +deleteReferrers +deleteUntagged detectManifestCollision dex discoverable DN Dockerfile +dryRun ExternalName Filebeat filesystem @@ -90,6 +94,8 @@ in-place io IoT JSON +json +keepTags KEP kubeconfig kubectl @@ -117,6 +123,8 @@ MERCHANTABILITY mgmt minify mkdocs +mostRecentlyPulledCount +mostRecentlyPushedCount mTLS mutex mutexes @@ -147,7 +155,9 @@ pprof PR PRs prometheus +pulledWithin pullPolicy +pushedWithin Qualys rc regclient @@ -206,6 +216,7 @@ twemoji UI unrecovered unreviewed +untagged uri URI url From c5c70748aa2e74c6eaf7d5ca2d996d5943fd93a3 Mon Sep 17 00:00:00 2001 From: mbshields Date: Tue, 24 Oct 2023 13:09:41 -0700 Subject: [PATCH 3/4] docs: add article for Retention Policies - spellcheck2 Signed-off-by: mbshields --- .wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.wordlist.txt b/.wordlist.txt index 71ef93b..2315bbd 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -160,6 +160,7 @@ pullPolicy pushedWithin Qualys rc +README regclient regctl replicaCount From a10fbffd7cbe62b516756b0b450cec181171620e Mon Sep 17 00:00:00 2001 From: mbshields Date: Wed, 1 Nov 2023 09:51:14 -0700 Subject: [PATCH 4/4] docs: add article for Retention Policies - resolve conflicting statements Signed-off-by: mbshields --- docs/articles/retention.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/articles/retention.md b/docs/articles/retention.md index 64794b2..49c33a1 100644 --- a/docs/articles/retention.md +++ b/docs/articles/retention.md @@ -19,7 +19,7 @@ Retention policies are configured in the `storage` section of the zot configurat By default, if no retention policies are defined, all tags are retained. -> :warning: If at least one retention policy is defined, all tags not matching any policy are removed. We recommend defining a default policy to avoid unintended removals. +> :warning: If at least one `keepTags` policy is defined for a repository, all tags not matching those policies are removed. To avoid unintended removals, we recommend defining a default policy, as described in [Configuration notes](#configuration-notes). ### Configuration example @@ -77,24 +77,19 @@ The following table lists the attributes available in the retention policy confi | pulledWithin | time | Retains the tags pulled during the last hours, such as 24h. | | patterns | regex | See Notes. | -> Mike's questions: -> - Which is the correct attribute name: `repositories` (as in config-retention.json) or `repoNames` (as in README.md)? -> - Are these statements conflicting? -> -> - If at least one retention policy is defined, all tags not matching any policy are removed. -> - If a repository matches no policy, the repository and all its tags are retained. ### Configuration notes - A repository will apply the first policy it matches. - If a repository matches no policy, the repository and all its tags are retained. +- If at least one `keepTags` policy is defined for a repository, all tags not matching those policies are removed. - If `keepTags` is present but empty, all tags are retained. - When multiple rules are configured, a tag is retained if it meets at least one rule. - When you specify a regex pattern combined with one or more rules, the rules are applied only to those tags matching the regex. - When you specify a regex pattern with no rules other than the default, all tags matching the pattern are retained. - In the repositories list, a single asterisk (/\*) matches all first-level items in the repository. A double asterisk (/*\*) matches all recursively. -:warning: If at least one retention policy is defined, all tags not matching any policy are removed. We recommend defining a default policy, such as the following example, as the last policy in the policy list. All tags that don't match the preceding policies will be retained by this default policy: +:warning: We recommend defining a default `keepTags` policy, such as the following example, as the last policy in the policy list. All tags that don't match the preceding policies will be retained by this default policy: _default policy example_