feat(retention): added image retention policies

feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <[email protected]>

errors/errors.go

@@ -163,4 +163,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrGCPolicyNotFound               = errors.New("gc: repo/tag policy not found")
 )

examples/README.md

@@ -84,6 +84,12 @@ to wasted storage, and background garbage collection can be enabled with:
         "gc": true,
 ```
 
+Orphan blobs are removed if they are older than gcDelay.
+
+```
+        "gcDelay": "2h"
+```
+
 It is also possible to store and serve images from multiple filesystems with
 their own repository paths, dedupe and garbage collection settings with:
 
@@ -106,6 +112,77 @@ their own repository paths, dedupe and garbage collection settings with:
     },
 ```
 
+## Retention
+
+You can define tag retention rules that govern how many tags of a given repository to retain, or for how long to retain certain tags.
+
+There are 4 possible rules for tags:
+
+mostRecentlyPushedCount: x - top x most recently pushed tags
+mostRecentlyPulledCount: x - top x most recently pulled tags
+pulledWithin: x hours - tags pulled in the last x hours
+pushedWithin: x hours - tags pushed in the last x hours
+
+If ANY of these rules are met by a tag, then it will be retained, in other words there is an OR logic between them
+
+repoNames uses glob patterns
+tag patterns uses regex
+
+```
+        "retention": {
+            "dryRun": false,  // if enabled will just log the retain action without actually removing
+            "delay": "24h",   // is applied on untagged and referrers, will remove them only if they are older than 24h
+            "policies": [     // a repo will match a policy if it matches any repoNames[] glob pattern, it will select the first policy it can matches
+                {
+                    "repoNames": ["infra/*", "prod/*"], // patterns to match
+                    "deleteReferrers": false,           // delete manifests with missing Subject (default is false)
+                    "deleteUntagged": true,             // delete untagged manifests (default is true)
+                    "KeepTags": [{                      // same as repo, the first pattern(this time regex) matched is the policy applied
+                        "patterns": ["v2.*", ".*-prod"] // if there is no rule then the default is to retain always, this tagRetention will retain all tags matching the regexes in the patterns list.
+                    },
+                    {
+                        "patterns": ["v3.*", ".*-prod"], 
+                        "pulledWithin": "168h"          // will keep v3.* and .*-prod tags that are pulled within last 168h
+                    }]
+                },                                      // all tags under infra/* and prod/* will be removed! because they don't match any retention policy
+                {
+                    "repoNames": ["tmp/**"],            // matches recursively all repos under tmp/
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "KeepTags": [{                      // will retain all tags starting with v1 and pulled within the last 168h
+                        "patterns": ["v1.*"],           // all the other tags will be removed
+                        "pulledWithin": "168h",      
+                        "pushedWithin": "168h"
+                    }]
+                },
+                {
+                    "repoNames": ["**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "keepTags": [{
+                        "mostRecentlyPushedCount": 10,    // top 10 recently pushed tags
+                        "mostRecentlyPulledCount": 10,    // top 10 recently pulled tags
+                        "pulledWithin": "720h",
+                        "pushedWithin": "720h"
+                    }]
+                }
+            ]
+        }
+```
+
+If a repo doesn't match any policy, then that repo and all its tags are retained. (default is to not delete anything)
+If keepTags is empty, then all tags are retained (default is to retain all tags)
+If we have at least one tagRetention policy in the tagRetention list then all tags that don't match at least one of them will be removed!
+
+For safety purpose you can have a default policy as the last policy in list, all tags that don't match the above policies will be retained by this one:
+```
+                    "keepTags": [
+                      {                               
+                        "patterns": [".*"]           // will retain all tags
+                      }
+                    }]
+```
+
 ## Authentication
 
 TLS mutual authentication and passphrase-based authentication are supported.

examples/config-gc-periodic.json

@@ -4,13 +4,13 @@
         "rootDirectory": "/tmp/zot",
         "gc": true,
         "gcDelay": "1h",
-        "gcInterval": "24h",
+        "gcInterval": "1h",
         "subPaths": {
             "/a": {
                 "rootDirectory": "/tmp/zot1",
                 "gc": true,
                 "gcDelay": "1h",
-                "gcInterval": "24h"
+                "gcInterval": "1h"
             }
         }
     },

examples/config-gc.json

@@ -3,9 +3,7 @@
     "storage": {
         "rootDirectory": "/tmp/zot",
         "gc": true,
-        "gcReferrers": true,
         "gcDelay": "2h",
-        "untaggedImageRetentionDelay": "4h",
         "gcInterval": "1h"
     },
     "http": {

examples/config-retention.json

@@ -0,0 +1,68 @@
+{
+    "distSpecVersion": "1.1.0-dev",
+    "storage": {
+        "rootDirectory": "/tmp/zot",
+        "gc": true,
+        "gcDelay": "2h",
+        "gcInterval": "1h",
+        "retention": {
+            "dryRun": false,
+            "delay": "24h",
+            "policies": [
+                {
+                    "repositories": ["infra/*", "prod/*"],
+                    "deleteReferrers": false,
+                    "keepTags": [{
+                        "patterns": ["v2.*", ".*-prod"]
+                    },
+                    {
+                        "patterns": ["v3.*", ".*-prod"],
+                        "pulledWithin": "168h"
+                    }]
+                },
+                {
+                    "repositories": ["tmp/**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "keepTags": [{
+                        "patterns": ["v1.*"],
+                        "pulledWithin": "168h",
+                        "pushedWithin": "168h"
+                    }]
+                },
+                {
+                    "repositories": ["**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "keepTags": [{
+                        "mostRecentlyPushedCount": 10,
+                        "mostRecentlyPulledCount": 10,
+                        "pulledWithin": "720h",
+                        "pushedWithin": "720h"
+                    }]
+                }
+            ]
+        },
+        "subPaths": {
+            "/a": {
+                "rootDirectory": "/tmp/zot1",
+                "dedupe": true,
+                "retention": {
+                    "policies": [
+                        {
+                            "repositories": ["infra/*", "prod/*"],
+                            "deleteReferrers": false
+                        }
+                    ]
+                }
+            }
+        }
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080"
+    },
+    "log": {
+        "level": "debug"
+    }
+}

examples/config-sync.json

@@ -35,12 +35,12 @@
 							}
 						},
 						{
-							"prefix": "/repo1/repo",
+							"prefix": "/repo2/repo",
 							"destination": "/repo",
 							"stripPrefix": true
 						},
 						{
-							"prefix": "/repo2/repo"
+							"prefix": "/repo3/**"
 						}
 					]
 				},
@@ -54,7 +54,7 @@
 					"onDemand": false,
 					"content": [
 						{
-							"prefix": "/repo2",
+							"prefix": "**",
 							"tags": {
 								"semver": true
 							}

pkg/api/config/config.go

@@ -23,17 +23,37 @@ var (
 )
 
 type StorageConfig struct {
-	RootDirectory               string
-	Dedupe                      bool
-	RemoteCache                 bool
-	GC                          bool
-	Commit                      bool
-	GCDelay                     time.Duration
-	GCInterval                  time.Duration
-	GCReferrers                 bool
-	UntaggedImageRetentionDelay time.Duration
-	StorageDriver               map[string]interface{} `mapstructure:",omitempty"`
-	CacheDriver                 map[string]interface{} `mapstructure:",omitempty"`
+	RootDirectory string
+	Dedupe        bool
+	RemoteCache   bool
+	GC            bool
+	Commit        bool
+	GCDelay       time.Duration // applied for blobs
+	GCInterval    time.Duration
+	Retention     ImageRetention
+	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
+	CacheDriver   map[string]interface{} `mapstructure:",omitempty"`
+}
+
+type ImageRetention struct {
+	DryRun   bool
+	Delay    time.Duration // applied for referrers and untagged
+	Policies []GCPolicy
+}
+
+type GCPolicy struct {
+	Repositories    []string
+	DeleteReferrers bool
+	DeleteUntagged  *bool
+	KeepTags        []KeepTagsPolicy
+}
+
+type KeepTagsPolicy struct {
+	Patterns                []string
+	PulledWithin            *time.Duration
+	PushedWithin            *time.Duration
+	MostRecentlyPushedCount int
+	MostRecentlyPulledCount int
 }
 
 type TLSConfig struct {
@@ -190,9 +210,11 @@ func New() *Config {
 		BinaryType:      BinaryType,
 		Storage: GlobalStorageConfig{
 			StorageConfig: StorageConfig{
-				GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay,
-				UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
-				GCInterval:                  storageConstants.DefaultGCInterval, Dedupe: true,
+				Dedupe:     true,
+				GC:         true,
+				GCDelay:    storageConstants.DefaultGCDelay,
+				GCInterval: storageConstants.DefaultGCInterval,
+				Retention:  ImageRetention{},
 			},
 		},
 		HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
@@ -368,6 +390,42 @@ func (c *Config) IsImageTrustEnabled() bool {
 	return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable
 }
 
+// check if tags retention is enabled.
+func (c *Config) IsRetentionEnabled() bool {
+	var needsMetaDB bool
+
+	for _, retentionPolicy := range c.Storage.Retention.Policies {
+		for _, tagRetentionPolicy := range retentionPolicy.KeepTags {
+			if c.isTagsRetentionEnabled(tagRetentionPolicy) {
+				needsMetaDB = true
+			}
+		}
+	}
+
+	for _, subpath := range c.Storage.SubPaths {
+		for _, retentionPolicy := range subpath.Retention.Policies {
+			for _, tagRetentionPolicy := range retentionPolicy.KeepTags {
+				if c.isTagsRetentionEnabled(tagRetentionPolicy) {
+					needsMetaDB = true
+				}
+			}
+		}
+	}
+
+	return needsMetaDB
+}
+
+func (c *Config) isTagsRetentionEnabled(tagRetentionPolicy KeepTagsPolicy) bool {
+	if tagRetentionPolicy.MostRecentlyPulledCount != 0 ||
+		tagRetentionPolicy.MostRecentlyPushedCount != 0 ||
+		tagRetentionPolicy.PulledWithin != nil ||
+		tagRetentionPolicy.PushedWithin != nil {
+		return true
+	}
+
+	return false
+}
+
 func (c *Config) IsCosignEnabled() bool {
 	return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign
 }